Repository: twitter/the-algorithm Branch: main Commit: c54bec0d4e02 Files: 7651 Total size: 26.3 MB Directory structure: gitextract_zmgfpzrq/ ├── .gitignore ├── COPYING ├── README.md ├── RETREIVAL_SIGNALS.md ├── ann/ │ └── src/ │ └── main/ │ ├── java/ │ │ └── com/ │ │ └── twitter/ │ │ └── ann/ │ │ ├── faiss/ │ │ │ ├── BUILD │ │ │ ├── NativeUtils.java │ │ │ └── swig/ │ │ │ ├── AlignedTableFloat32.java │ │ │ ├── AlignedTableUint16.java │ │ │ ├── AlignedTableUint8.java │ │ │ ├── ArrayInvertedLists.java │ │ │ ├── AutoTuneCriterion.java │ │ │ ├── BUILD │ │ │ ├── BitstringReader.java │ │ │ ├── BitstringWriter.java │ │ │ ├── BufferList.java │ │ │ ├── ByteVector.java │ │ │ ├── ByteVectorVector.java │ │ │ ├── CenteringTransform.java │ │ │ ├── CharVector.java │ │ │ ├── Clustering.java │ │ │ ├── Clustering1D.java │ │ │ ├── ClusteringIterationStats.java │ │ │ ├── ClusteringParameters.java │ │ │ ├── DistanceComputer.java │ │ │ ├── DoubleVector.java │ │ │ ├── FloatVector.java │ │ │ ├── FloatVectorVector.java │ │ │ ├── GenHammingComputer16.java │ │ │ ├── GenHammingComputer32.java │ │ │ ├── GenHammingComputer8.java │ │ │ ├── GenHammingComputerM8.java │ │ │ ├── HNSW.java │ │ │ ├── HNSWStats.java │ │ │ ├── HStackInvertedLists.java │ │ │ ├── HammingComputer16.java │ │ │ ├── HammingComputer20.java │ │ │ ├── HammingComputer32.java │ │ │ ├── HammingComputer4.java │ │ │ ├── HammingComputer64.java │ │ │ ├── HammingComputer8.java │ │ │ ├── HammingComputerDefault.java │ │ │ ├── HammingComputerM4.java │ │ │ ├── HammingComputerM8.java │ │ │ ├── IDSelector.java │ │ │ ├── IDSelectorArray.java │ │ │ ├── IDSelectorBatch.java │ │ │ ├── IDSelectorRange.java │ │ │ ├── ITQMatrix.java │ │ │ ├── ITQTransform.java │ │ │ ├── IVFPQSearchParameters.java │ │ │ ├── IVFSearchParameters.java │ │ │ ├── Index.java │ │ │ ├── Index2Layer.java │ │ │ ├── IndexBinary.java │ │ │ ├── IndexBinaryFlat.java │ │ │ ├── IndexBinaryFromFloat.java │ │ │ ├── IndexBinaryHNSW.java │ │ │ ├── IndexBinaryIVF.java │ │ │ ├── IndexFlat.java │ │ │ ├── IndexFlat1D.java │ │ │ ├── IndexFlatCodes.java │ │ │ ├── IndexFlatIP.java │ │ │ ├── IndexFlatL2.java │ │ │ ├── IndexHNSW.java │ │ │ ├── IndexHNSW2Level.java │ │ │ ├── IndexHNSWFlat.java │ │ │ ├── IndexHNSWPQ.java │ │ │ ├── IndexHNSWSQ.java │ │ │ ├── IndexIDMap.java │ │ │ ├── IndexIVF.java │ │ │ ├── IndexIVFFlat.java │ │ │ ├── IndexIVFFlatDedup.java │ │ │ ├── IndexIVFPQ.java │ │ │ ├── IndexIVFPQStats.java │ │ │ ├── IndexIVFScalarQuantizer.java │ │ │ ├── IndexIVFStats.java │ │ │ ├── IndexLSH.java │ │ │ ├── IndexPQ.java │ │ │ ├── IndexPQStats.java │ │ │ ├── IndexRefine.java │ │ │ ├── IndexRefineFlat.java │ │ │ ├── IndexScalarQuantizer.java │ │ │ ├── IndexShards.java │ │ │ ├── IndexSplitVectors.java │ │ │ ├── IntVector.java │ │ │ ├── InterruptCallback.java │ │ │ ├── IntersectionCriterion.java │ │ │ ├── InvertedLists.java │ │ │ ├── InvertedListsPtrVector.java │ │ │ ├── Level1Quantizer.java │ │ │ ├── LinearTransform.java │ │ │ ├── LongVector.java │ │ │ ├── LongVectorVector.java │ │ │ ├── MapLong2Long.java │ │ │ ├── MaskedInvertedLists.java │ │ │ ├── MetricType.java │ │ │ ├── MultiIndexQuantizer.java │ │ │ ├── MultiIndexQuantizer2.java │ │ │ ├── NormalizationTransform.java │ │ │ ├── OPQMatrix.java │ │ │ ├── OnDiskInvertedLists.java │ │ │ ├── OnDiskInvertedListsIOHook.java │ │ │ ├── OnDiskOneList.java │ │ │ ├── OneRecallAtRCriterion.java │ │ │ ├── OperatingPoint.java │ │ │ ├── OperatingPointVector.java │ │ │ ├── OperatingPoints.java │ │ │ ├── PCAMatrix.java │ │ │ ├── PQDecoder16.java │ │ │ ├── PQDecoder8.java │ │ │ ├── PQDecoderGeneric.java │ │ │ ├── PQEncoder16.java │ │ │ ├── PQEncoder8.java │ │ │ ├── PQEncoderGeneric.java │ │ │ ├── ParameterRange.java │ │ │ ├── ParameterSpace.java │ │ │ ├── PartitionStats.java │ │ │ ├── PermutationObjective.java │ │ │ ├── PolysemousTraining.java │ │ │ ├── ProductQuantizer.java │ │ │ ├── ProgressiveDimClustering.java │ │ │ ├── ProgressiveDimClusteringParameters.java │ │ │ ├── ProgressiveDimIndexFactory.java │ │ │ ├── RandomRotationMatrix.java │ │ │ ├── RangeQueryResult.java │ │ │ ├── RangeSearchPartialResult.java │ │ │ ├── RangeSearchResult.java │ │ │ ├── ReadOnlyInvertedLists.java │ │ │ ├── ReconstructFromNeighbors.java │ │ │ ├── RemapDimensionsTransform.java │ │ │ ├── ReproduceDistancesObjective.java │ │ │ ├── SWIGTYPE_p_AlignedTableT_float_32_t.java │ │ │ ├── SWIGTYPE_p_AlignedTableT_float_t.java │ │ │ ├── SWIGTYPE_p_DirectMap.java │ │ │ ├── SWIGTYPE_p_DirectMap__Type.java │ │ │ ├── SWIGTYPE_p_FILE.java │ │ │ ├── SWIGTYPE_p_IOReader.java │ │ │ ├── SWIGTYPE_p_IOWriter.java │ │ │ ├── SWIGTYPE_p_ScalarQuantizer.java │ │ │ ├── SWIGTYPE_p_ScalarQuantizer__QuantizerType.java │ │ │ ├── SWIGTYPE_p_double.java │ │ │ ├── SWIGTYPE_p_faiss__AlignedTableTightAllocT_float_32_t.java │ │ │ ├── SWIGTYPE_p_faiss__AlignedTableTightAllocT_uint16_t_32_t.java │ │ │ ├── SWIGTYPE_p_faiss__AlignedTableTightAllocT_unsigned_char_32_t.java │ │ │ ├── SWIGTYPE_p_faiss__BinaryInvertedListScanner.java │ │ │ ├── SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_float_int64_t_t_t.java │ │ │ ├── SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_int_int64_t_t_t.java │ │ │ ├── SWIGTYPE_p_faiss__HeapArrayT_faiss__CMinT_float_int64_t_t_t.java │ │ │ ├── SWIGTYPE_p_faiss__IOReader.java │ │ │ ├── SWIGTYPE_p_faiss__IOWriter.java │ │ │ ├── SWIGTYPE_p_faiss__InvertedListScanner.java │ │ │ ├── SWIGTYPE_p_faiss__LockLevels.java │ │ │ ├── SWIGTYPE_p_faiss__OnDiskInvertedLists__OngoingPrefetch.java │ │ │ ├── SWIGTYPE_p_faiss__RandomGenerator.java │ │ │ ├── SWIGTYPE_p_float.java │ │ │ ├── SWIGTYPE_p_int.java │ │ │ ├── SWIGTYPE_p_long.java │ │ │ ├── SWIGTYPE_p_long_long.java │ │ │ ├── SWIGTYPE_p_omp_lock_t.java │ │ │ ├── SWIGTYPE_p_p_faiss__Index.java │ │ │ ├── SWIGTYPE_p_p_faiss__InvertedLists.java │ │ │ ├── SWIGTYPE_p_p_faiss__VectorTransform.java │ │ │ ├── SWIGTYPE_p_std__listT_faiss__OnDiskInvertedLists__Slot_t.java │ │ │ ├── SWIGTYPE_p_std__pairT_float_int_t.java │ │ │ ├── SWIGTYPE_p_std__priority_queueT_faiss__HNSW__NodeDistFarther_t.java │ │ │ ├── SWIGTYPE_p_std__priority_queueT_std__pairT_float_int_t_t.java │ │ │ ├── SWIGTYPE_p_std__unordered_mapT_long_long_t.java │ │ │ ├── SWIGTYPE_p_std__unordered_multimapT_int64_t_int64_t_t.java │ │ │ ├── SWIGTYPE_p_std__vectorT_faiss__BufferList__Buffer_t.java │ │ │ ├── SWIGTYPE_p_std__vectorT_faiss__ClusteringIterationStats_t.java │ │ │ ├── SWIGTYPE_p_std__vectorT_faiss__HNSW__NodeDistFarther_t.java │ │ │ ├── SWIGTYPE_p_std__vectorT_faiss__Index_p_t.java │ │ │ ├── SWIGTYPE_p_std__vectorT_faiss__InvertedLists_const_p_t.java │ │ │ ├── SWIGTYPE_p_std__vectorT_faiss__OnDiskOneList_t.java │ │ │ ├── SWIGTYPE_p_std__vectorT_faiss__ParameterRange_t.java │ │ │ ├── SWIGTYPE_p_std__vectorT_faiss__RangeQueryResult_t.java │ │ │ ├── SWIGTYPE_p_std__vectorT_faiss__RangeSearchPartialResult_p_t.java │ │ │ ├── SWIGTYPE_p_std__vectorT_int64_t_t.java │ │ │ ├── SWIGTYPE_p_std__vectorT_long_t.java │ │ │ ├── SWIGTYPE_p_std__vectorT_omp_lock_t_t.java │ │ │ ├── SWIGTYPE_p_std__vectorT_std__vectorT_int64_t_t_t.java │ │ │ ├── SWIGTYPE_p_std__vectorT_std__vectorT_unsigned_long_t_t.java │ │ │ ├── SWIGTYPE_p_uint16_t.java │ │ │ ├── SWIGTYPE_p_uint32_t.java │ │ │ ├── SWIGTYPE_p_unsigned_char.java │ │ │ ├── SWIGTYPE_p_unsigned_long.java │ │ │ ├── SWIGTYPE_p_void.java │ │ │ ├── SimulatedAnnealingOptimizer.java │ │ │ ├── SimulatedAnnealingParameters.java │ │ │ ├── SliceInvertedLists.java │ │ │ ├── SlidingIndexWindow.java │ │ │ ├── StopWordsInvertedLists.java │ │ │ ├── Uint64Vector.java │ │ │ ├── VStackInvertedLists.java │ │ │ ├── VectorTransform.java │ │ │ ├── VectorTransformVector.java │ │ │ ├── VisitedTable.java │ │ │ ├── doubleArray.java │ │ │ ├── floatArray.java │ │ │ ├── float_maxheap_array_t.java │ │ │ ├── float_minheap_array_t.java │ │ │ ├── intArray.java │ │ │ ├── int_maxheap_array_t.java │ │ │ ├── int_minheap_array_t.java │ │ │ ├── longArray.java │ │ │ ├── resources/ │ │ │ │ ├── .gitignore │ │ │ │ ├── .gitkeep │ │ │ │ └── BUILD │ │ │ ├── swigfaiss.java │ │ │ ├── swigfaissConstants.java │ │ │ └── swigfaissJNI.java │ │ └── hnsw/ │ │ ├── BUILD │ │ ├── DistanceFunction.java │ │ ├── DistancedItem.java │ │ ├── DistancedItemQueue.java │ │ ├── HnswIndex.java │ │ ├── HnswIndexIOUtil.java │ │ ├── HnswMeta.java │ │ ├── HnswNode.java │ │ └── IllegalDuplicateInsertException.java │ ├── python/ │ │ └── dataflow/ │ │ ├── BUILD.bazel │ │ ├── bq.sql │ │ ├── faiss_index_bq_dataset.py │ │ └── worker_harness/ │ │ ├── Dockerfile │ │ └── cloudbuild.yml │ ├── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── ann/ │ │ ├── annoy/ │ │ │ ├── AnnoyCommon.scala │ │ │ ├── BUILD │ │ │ ├── RawAnnoyIndexBuilder.scala │ │ │ ├── RawAnnoyQueryIndex.scala │ │ │ ├── TypedAnnoyIndex.scala │ │ │ ├── TypedAnnoyIndexBuilderWithFile.scala │ │ │ └── TypedAnnoyQueryIndexWithFile.scala │ │ ├── brute_force/ │ │ │ ├── BUILD │ │ │ ├── BruteForceDeserialization.scala │ │ │ └── BruteForceIndex.scala │ │ ├── common/ │ │ │ ├── AnnInjections.scala │ │ │ ├── Api.scala │ │ │ ├── BUILD │ │ │ ├── EmbeddingProducer.scala │ │ │ ├── IndexOutputFile.scala │ │ │ ├── IndexTransformer.scala │ │ │ ├── MemoizedInEpochs.scala │ │ │ ├── Metric.scala │ │ │ ├── QueryableById.scala │ │ │ ├── QueryableByIdImplementation.scala │ │ │ ├── QueryableOperations.scala │ │ │ ├── ReadWriteFuturePool.scala │ │ │ ├── Serialization.scala │ │ │ ├── ServiceClientQueryable.scala │ │ │ ├── ShardApi.scala │ │ │ ├── ShardedSerialization.scala │ │ │ └── Task.scala │ │ ├── dataflow/ │ │ │ └── offline/ │ │ │ ├── ANNIndexBuilderBeamJob.scala │ │ │ ├── BUILD │ │ │ ├── BaseEmbeddingData.scala │ │ │ ├── FlatEmbeddingData.scala │ │ │ └── GroupedEmbeddingData.scala │ │ ├── experimental/ │ │ │ ├── BUILD.bazel │ │ │ └── Runner.scala │ │ ├── faiss/ │ │ │ ├── BUILD │ │ │ ├── FaissCommon.scala │ │ │ ├── FaissIndex.scala │ │ │ ├── FaissIndexer.scala │ │ │ ├── HourlyDirectoryWithSuccessFileListing.scala │ │ │ ├── HourlyShardedIndex.scala │ │ │ └── QueryableIndexAdapter.scala │ │ ├── featurestore/ │ │ │ ├── BUILD │ │ │ └── FeatureStoreEmbeddingProducer.scala │ │ ├── file_store/ │ │ │ ├── BUILD │ │ │ ├── ReadableIndexIdFileStore.scala │ │ │ └── WritableIndexIdFileStore.scala │ │ ├── hnsw/ │ │ │ ├── BUILD │ │ │ ├── DistanceFunctionGenerator.scala │ │ │ ├── Hnsw.scala │ │ │ ├── HnswCommon.scala │ │ │ ├── HnswIOUtil.scala │ │ │ ├── IdEmbeddingMap.scala │ │ │ ├── JMapBasedIdEmbeddingMap.scala │ │ │ ├── MapDbBasedIdEmbeddingMap.scala │ │ │ ├── SerializableHnsw.scala │ │ │ └── TypedHnswIndex.scala │ │ ├── manhattan/ │ │ │ ├── BUILD │ │ │ ├── ManhattanEmbeddingProducer.scala │ │ │ └── README │ │ ├── scalding/ │ │ │ ├── benchmark/ │ │ │ │ ├── BUILD │ │ │ │ └── Knn.scala │ │ │ └── offline/ │ │ │ ├── BUILD.bazel │ │ │ ├── IndexingStrategy.scala │ │ │ ├── KnnDebug.scala │ │ │ ├── KnnEntityRecoDebugJob.scala │ │ │ ├── KnnHelper.scala │ │ │ ├── KnnOfflineJob.scala │ │ │ ├── KnnTruthSetGenerator.scala │ │ │ ├── ParameterlessQueryable.scala │ │ │ ├── README │ │ │ ├── faissindexbuilder/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── IndexBuilder.scala │ │ │ │ └── IndexBuilderApp.scala │ │ │ ├── indexbuilder/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── IndexBuilder.scala │ │ │ │ ├── IndexBuilderApp.scala │ │ │ │ └── README.rst │ │ │ └── indexbuilderfrombq/ │ │ │ ├── BUILD.bazel │ │ │ ├── IndexBuilderFromBQ.scala │ │ │ └── IndexBuilderFromBQApp.scala │ │ ├── serialization/ │ │ │ ├── BUILD │ │ │ ├── DummyANNIndexInjection.scala │ │ │ ├── PersistedEmbeddingInjection.scala │ │ │ └── ThriftIteratorIO.scala │ │ ├── service/ │ │ │ ├── loadtest/ │ │ │ │ ├── AnnLoadTest.scala │ │ │ │ ├── AnnLoadTestMain.scala │ │ │ │ ├── AnnLoadTestWorker.scala │ │ │ │ ├── BUILD │ │ │ │ ├── EmbeddingIndexer.scala │ │ │ │ ├── LoadTestRecorder.scala │ │ │ │ ├── LoadTestUtils.scala │ │ │ │ └── README.md │ │ │ └── query_server/ │ │ │ ├── common/ │ │ │ │ ├── BUILD │ │ │ │ ├── BaseQueryIndexServer.scala │ │ │ │ ├── Exceptions.scala │ │ │ │ ├── FaissIndexPathProvider.scala │ │ │ │ ├── IndexPathProvider.scala │ │ │ │ ├── QueryIndexThriftController.scala │ │ │ │ ├── QueryServerUtil.scala │ │ │ │ ├── QueryableProvider.scala │ │ │ │ ├── RefreshableQueryable.scala │ │ │ │ ├── UnsafeQueryIndexServer.scala │ │ │ │ ├── throttling/ │ │ │ │ │ ├── AuroraCPUStatsReader.scala │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── ThrottlingBasedQualityTask.scala │ │ │ │ │ ├── WindowedStats.scala │ │ │ │ │ └── WindowedThrottlingInstrument.scala │ │ │ │ └── warmup/ │ │ │ │ ├── BUILD │ │ │ │ └── Warmup.scala │ │ │ ├── faiss/ │ │ │ │ ├── BUILD │ │ │ │ └── FaissQueryIndexServer.scala │ │ │ └── hnsw/ │ │ │ ├── BUILD │ │ │ └── HnswQueryIndexServer.scala │ │ └── util/ │ │ ├── BUILD │ │ └── IndexBuilderUtils.scala │ └── thrift/ │ └── com/ │ └── twitter/ │ └── ann/ │ ├── common/ │ │ ├── BUILD │ │ └── ann_common.thrift │ ├── knn/ │ │ ├── BUILD │ │ └── knn.thrift │ └── serialization/ │ ├── BUILD │ └── serialization.thrift ├── ci/ │ └── ci.sh ├── cr-mixer/ │ ├── BUILD.bazel │ ├── README.md │ ├── server/ │ │ └── src/ │ │ └── main/ │ │ ├── resources/ │ │ │ ├── BUILD.bazel │ │ │ ├── config/ │ │ │ │ └── decider.yml │ │ │ └── logback.xml │ │ └── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── cr_mixer/ │ │ ├── BUILD.bazel │ │ ├── CrMixerHttpServerWarmupHandler.scala │ │ ├── CrMixerServer.scala │ │ ├── CrMixerThriftServerWarmupHandler.scala │ │ ├── blender/ │ │ │ ├── AdsBlender.scala │ │ │ ├── BUILD │ │ │ ├── BlendedCandidatesBuilder.scala │ │ │ ├── ContentSignalBlender.scala │ │ │ ├── CountWeightedInterleaveBlender.scala │ │ │ ├── InterleaveBlender.scala │ │ │ ├── SourceTypeBackFillBlender.scala │ │ │ └── SwitchBlender.scala │ │ ├── candidate_generation/ │ │ │ ├── AdsCandidateGenerator.scala │ │ │ ├── AdsCandidateSourcesRouter.scala │ │ │ ├── BUILD │ │ │ ├── CandidateSourcesRouter.scala │ │ │ ├── CrCandidateGenerator.scala │ │ │ ├── CustomizedRetrievalCandidateGeneration.scala │ │ │ ├── FrsTweetCandidateGenerator.scala │ │ │ ├── RelatedTweetCandidateGenerator.scala │ │ │ ├── RelatedVideoTweetCandidateGenerator.scala │ │ │ ├── SimClustersInterestedInCandidateGeneration.scala │ │ │ ├── TopicTweetCandidateGenerator.scala │ │ │ └── UtegTweetCandidateGenerator.scala │ │ ├── config/ │ │ │ ├── BUILD │ │ │ ├── SimClustersANNConfig.scala │ │ │ └── TimeoutConfig.scala │ │ ├── controller/ │ │ │ ├── BUILD.bazel │ │ │ └── CrMixerThriftController.scala │ │ ├── exception/ │ │ │ ├── BUILD │ │ │ └── InvalidSANNConfigException.scala │ │ ├── featureswitch/ │ │ │ ├── BUILD │ │ │ ├── CrMixerLoggingABDecider.scala │ │ │ ├── ParamsBuilder.scala │ │ │ └── SetImpressedBucketsLocalContextFilter.scala │ │ ├── filter/ │ │ │ ├── BUILD │ │ │ ├── FilterBase.scala │ │ │ ├── ImpressedTweetlistFilter.scala │ │ │ ├── InNetworkFilter.scala │ │ │ ├── PostRankFilterRunner.scala │ │ │ ├── PreRankFilterRunner.scala │ │ │ ├── ReplyFilter.scala │ │ │ ├── RetweetFilter.scala │ │ │ ├── TweetAgeFilter.scala │ │ │ ├── TweetInfoHealthFilterBase.scala │ │ │ ├── UtegFilterRunner.scala │ │ │ ├── UtegHealthFilter.scala │ │ │ └── VideoTweetFilter.scala │ │ ├── logging/ │ │ │ ├── AdsRecommendationsScribeLogger.scala │ │ │ ├── BUILD │ │ │ ├── CrMixerScribeLogger.scala │ │ │ ├── RelatedTweetScribeLogger.scala │ │ │ ├── ScribeLoggerUtils.scala │ │ │ ├── ScribeMetadata.scala │ │ │ ├── TopLevelDdgMetricsMetadata.scala │ │ │ └── UtegTweetScribeLogger.scala │ │ ├── model/ │ │ │ ├── BUILD │ │ │ ├── Candidate.scala │ │ │ ├── CandidateGenerationInfo.scala │ │ │ ├── CandidateGeneratorQuery.scala │ │ │ ├── EarlybirdSimilarityEngineType.scala │ │ │ ├── HealthThreshold.scala │ │ │ ├── ModelConfig.scala │ │ │ ├── ModuleNames.scala │ │ │ ├── TopicTweetWithScore.scala │ │ │ ├── TweetWithAuthor.scala │ │ │ ├── TweetWithScore.scala │ │ │ └── TweetWithScoreAndSocialProof.scala │ │ ├── module/ │ │ │ ├── ActivePromotedTweetStoreModule.scala │ │ │ ├── BUILD.bazel │ │ │ ├── BlueVerifiedAnnotationStoreModule.scala │ │ │ ├── CertoStratoStoreModule.scala │ │ │ ├── ConsumersBasedUserAdGraphStoreModule.scala │ │ │ ├── ConsumersBasedUserTweetGraphStoreModule.scala │ │ │ ├── ConsumersBasedUserVideoGraphStoreModule.scala │ │ │ ├── CrMixerParamConfigModule.scala │ │ │ ├── DiffusionStoreModule.scala │ │ │ ├── EarlybirdRecencyBasedCandidateStoreModule.scala │ │ │ ├── EmbeddingStoreModule.scala │ │ │ ├── FrsStoreModule.scala │ │ │ ├── MHMtlsParamsModule.scala │ │ │ ├── OfflineCandidateStoreModule.scala │ │ │ ├── RealGraphOonStoreModule.scala │ │ │ ├── RealGraphStoreMhModule.scala │ │ │ ├── RepresentationManagerModule.scala │ │ │ ├── RepresentationScorerModule.scala │ │ │ ├── SampleSimilarityEngineModule.scala │ │ │ ├── SimClustersANNServiceNameToClientMapper.scala │ │ │ ├── SkitStratoStoreModule.scala │ │ │ ├── StrongTiePredictionStoreModule.scala │ │ │ ├── TripCandidateStoreModule.scala │ │ │ ├── TweetInfoStoreModule.scala │ │ │ ├── TweetRecentEngagedUserStoreModule.scala │ │ │ ├── TweetRecommendationResultsStoreModule.scala │ │ │ ├── TwhinCollabFilterStratoStoreModule.scala │ │ │ ├── TwiceClustersMembersStoreModule.scala │ │ │ ├── UnifiedCacheClient.scala │ │ │ ├── UserSignalServiceColumnModule.scala │ │ │ ├── UserSignalServiceStoreModule.scala │ │ │ ├── UserStateStoreModule.scala │ │ │ ├── core/ │ │ │ │ ├── ABDeciderModule.scala │ │ │ │ ├── CrMixerFlagModule.scala │ │ │ │ ├── CrMixerLoggingABDeciderModule.scala │ │ │ │ ├── FeatureContextBuilderModule.scala │ │ │ │ ├── FeatureSwitchesModule.scala │ │ │ │ ├── KafkaProducerModule.scala │ │ │ │ ├── LoggerFactoryModule.scala │ │ │ │ ├── MemoizingStatsReceiverModule.scala │ │ │ │ └── TimeoutConfigModule.scala │ │ │ ├── grpc_client/ │ │ │ │ └── NaviGRPCClientModule.scala │ │ │ ├── similarity_engine/ │ │ │ │ ├── CertoTopicTweetSimilarityEngineModule.scala │ │ │ │ ├── ConsumerBasedWalsSimilarityEngineModule.scala │ │ │ │ ├── ConsumerEmbeddingBasedTripSimilarityEngineModule.scala │ │ │ │ ├── ConsumerEmbeddingBasedTwHINSimilarityEngineModule.scala │ │ │ │ ├── ConsumerEmbeddingBasedTwoTowerSimilarityEngineModule.scala │ │ │ │ ├── ConsumersBasedUserAdGraphSimilarityEngineModule.scala │ │ │ │ ├── ConsumersBasedUserVideoGraphSimilarityEngineModule.scala │ │ │ │ ├── DiffusionBasedSimilarityEngineModule.scala │ │ │ │ ├── EarlybirdSimilarityEngineModule.scala │ │ │ │ ├── ProducerBasedUnifiedSimilarityEngineModule.scala │ │ │ │ ├── ProducerBasedUserAdGraphSimilarityEngineModule.scala │ │ │ │ ├── ProducerBasedUserTweetGraphSimilarityEngineModule.scala │ │ │ │ ├── SimClustersANNSimilarityEngineModule.scala │ │ │ │ ├── SkitTopicTweetSimilarityEngineModule.scala │ │ │ │ ├── TweetBasedQigSimilarityEngineModule.scala │ │ │ │ ├── TweetBasedTwHINSimlarityEngineModule.scala │ │ │ │ ├── TweetBasedUnifiedSimilarityEngineModule.scala │ │ │ │ ├── TweetBasedUserAdGraphSimilarityEngineModule.scala │ │ │ │ ├── TweetBasedUserTweetGraphSimilarityEngineModule.scala │ │ │ │ ├── TweetBasedUserVideoGraphSimilarityEngineModule.scala │ │ │ │ ├── TwhinCollabFilterLookupSimilarityEngineModule.scala │ │ │ │ └── UserTweetEntityGraphSimilarityEngineModule.scala │ │ │ └── thrift_client/ │ │ │ ├── AnnQueryServiceClientModule.scala │ │ │ ├── EarlybirdSearchClientModule.scala │ │ │ ├── FrsClientModule.scala │ │ │ ├── HydraPartitionClientModule.scala │ │ │ ├── HydraRootClientModule.scala │ │ │ ├── QigServiceClientModule.scala │ │ │ ├── SimClustersAnnServiceClientModule.scala │ │ │ ├── TweetyPieClientModule.scala │ │ │ ├── UserAdGraphClientModule.scala │ │ │ ├── UserTweetEntityGraphClientModule.scala │ │ │ ├── UserTweetGraphClientModule.scala │ │ │ ├── UserTweetGraphPlusClientModule.scala │ │ │ └── UserVideoGraphClientModule.scala │ │ ├── param/ │ │ │ ├── AdsParams.scala │ │ │ ├── BUILD │ │ │ ├── BlenderParams.scala │ │ │ ├── BypassInterleaveAndRankParams.scala │ │ │ ├── ConsumerBasedWalsParams.scala │ │ │ ├── ConsumerEmbeddingBasedCandidateGenerationParams.scala │ │ │ ├── ConsumerEmbeddingBasedTripParams.scala │ │ │ ├── ConsumerEmbeddingBasedTwHINParams.scala │ │ │ ├── ConsumerEmbeddingBasedTwoTowerParams.scala │ │ │ ├── ConsumersBasedUserAdGraphParams.scala │ │ │ ├── ConsumersBasedUserTweetGraphParams.scala │ │ │ ├── ConsumersBasedUserVideoGraphParams.scala │ │ │ ├── CrMixerParamConfig.scala │ │ │ ├── CustomizedRetrievalBasedCandidateGenerationParams.scala │ │ │ ├── CustomizedRetrievalBasedFTROfflineInterestedInParams.scala │ │ │ ├── CustomizedRetrievalBasedOfflineInterestedInParams.scala │ │ │ ├── CustomizedRetrievalBasedTwhinParams.scala │ │ │ ├── EarlybirdFrsBasedCandidateGenerationParams.scala │ │ │ ├── FrsParams.scala │ │ │ ├── GlobalParams.scala │ │ │ ├── GoodProfileClickParams.scala │ │ │ ├── GoodTweetClickParams.scala │ │ │ ├── InterestedInParams.scala │ │ │ ├── ProducerBasedCandidateGenerationParams.scala │ │ │ ├── ProducerBasedUserAdGraphParams.scala │ │ │ ├── ProducerBasedUserTweetGraphParams.scala │ │ │ ├── RankerParams.scala │ │ │ ├── RealGraphInParams.scala │ │ │ ├── RealGraphOonParams.scala │ │ │ ├── RecentFollowsParams.scala │ │ │ ├── RecentNegativeSignalParams.scala │ │ │ ├── RecentNotificationsParams.scala │ │ │ ├── RecentOriginalTweetsParams.scala │ │ │ ├── RecentReplyTweetsParams.scala │ │ │ ├── RecentRetweetsParams.scala │ │ │ ├── RecentTweetFavoritesParams.scala │ │ │ ├── RelatedTweetGlobalParams.scala │ │ │ ├── RelatedTweetProducerBasedParams.scala │ │ │ ├── RelatedTweetTweetBasedParams.scala │ │ │ ├── RelatedVideoTweetGlobalParams.scala │ │ │ ├── RelatedVideoTweetTweetBasedParams.scala │ │ │ ├── RepeatedProfileVisitsParams.scala │ │ │ ├── SimClustersANNParams.scala │ │ │ ├── TopicTweetParams.scala │ │ │ ├── TweetBasedCandidateGenerationParams.scala │ │ │ ├── TweetBasedTwHINParams.scala │ │ │ ├── TweetBasedUserAdGraphParams.scala │ │ │ ├── TweetBasedUserTweetGraphParams.scala │ │ │ ├── TweetBasedUserVideoGraphParams.scala │ │ │ ├── TweetSharesParams.scala │ │ │ ├── UnifiedSETweetCombinationMethod.scala │ │ │ ├── UnifiedUSSSignalParams.scala │ │ │ ├── UtegTweetGlobalParams.scala │ │ │ ├── VideoTweetFilterParams.scala │ │ │ ├── VideoViewTweetsParams.scala │ │ │ └── decider/ │ │ │ ├── BUILD │ │ │ ├── CrMixerDecider.scala │ │ │ ├── DeciderKey.scala │ │ │ └── EndpointLoadShedder.scala │ │ ├── ranker/ │ │ │ ├── BUILD │ │ │ ├── DefaultRanker.scala │ │ │ └── SwitchRanker.scala │ │ ├── scribe/ │ │ │ ├── BUILD │ │ │ └── ScribeCategory.scala │ │ ├── service/ │ │ │ ├── BUILD.bazel │ │ │ └── CrMixerAlertNotificationConfig.scala │ │ ├── similarity_engine/ │ │ │ ├── BUILD │ │ │ ├── CertoTopicTweetSimilarityEngine.scala │ │ │ ├── ConsumerBasedWalsSimilarityEngine.scala │ │ │ ├── ConsumerEmbeddingBasedTripSimilarityEngine.scala │ │ │ ├── ConsumerEmbeddingBasedTwHINSimilarityEngine.scala │ │ │ ├── ConsumerEmbeddingBasedTwoTowerSimilarityEngine.scala │ │ │ ├── ConsumersBasedUserAdGraphSimilarityEngine.scala │ │ │ ├── ConsumersBasedUserVideoGraphSimilarityEngine.scala │ │ │ ├── DiffusionBasedSimilarityEngine.scala │ │ │ ├── EarlybirdModelBasedSimilarityEngine.scala │ │ │ ├── EarlybirdRecencyBasedSimilarityEngine.scala │ │ │ ├── EarlybirdSimilarityEngine.scala │ │ │ ├── EarlybirdSimilarityEngineBase.scala │ │ │ ├── EarlybirdSimilarityEngineRouter.scala │ │ │ ├── EarlybirdTensorflowBasedSimilarityEngine.scala │ │ │ ├── FilterUtil.scala │ │ │ ├── HnswANNSimilarityEngine.scala │ │ │ ├── LookupSimilarityEngine.scala │ │ │ ├── ModelBasedANNStore.scala │ │ │ ├── ProducerBasedUnifiedSimilarityEngine.scala │ │ │ ├── ProducerBasedUserAdGraphSimilarityEngine.scala │ │ │ ├── ProducerBasedUserTweetGraphSimilarityEngine.scala │ │ │ ├── SimClustersANNSimilarityEngine.scala │ │ │ ├── SimilarityEngine.scala │ │ │ ├── SimilaritySourceOrderingUtil.scala │ │ │ ├── SkitHighPrecisionTopicTweetSimilarityEngine.scala │ │ │ ├── SkitTopicTweetSimilarityEngine.scala │ │ │ ├── StandardSimilarityEngine.scala │ │ │ ├── TweetBasedQigSimilarityEngine.scala │ │ │ ├── TweetBasedUnifiedSimilarityEngine.scala │ │ │ ├── TweetBasedUserAdGraphSimilarityEngine.scala │ │ │ ├── TweetBasedUserTweetGraphSimilarityEngine.scala │ │ │ ├── TweetBasedUserVideoGraphSimilarityEngine.scala │ │ │ ├── TwhinCollabFilterSimilarityEngine.scala │ │ │ └── UserTweetEntityGraphSimilarityEngine.scala │ │ ├── source_signal/ │ │ │ ├── BUILD │ │ │ ├── FrsSourceGraphFetcher.scala │ │ │ ├── FrsSourceSignalFetcher.scala │ │ │ ├── FrsStore.scala │ │ │ ├── RealGraphInSourceGraphFetcher.scala │ │ │ ├── RealGraphOonSourceGraphFetcher.scala │ │ │ ├── SourceFetcher.scala │ │ │ ├── SourceGraphFetcher.scala │ │ │ ├── SourceInfoRouter.scala │ │ │ ├── SourceSignalFetcher.scala │ │ │ ├── UssSourceSignalFetcher.scala │ │ │ └── UssStore.scala │ │ └── util/ │ │ ├── BUILD │ │ ├── CandidateGenerationKeyUtil.scala │ │ ├── CountWeightedInterleaveUtil.scala │ │ ├── EarlybirdSearchUtil.scala │ │ ├── InterleaveUtil.scala │ │ ├── MetricTagUtil.scala │ │ └── SignalTimestampStatsUtil.scala │ └── thrift/ │ └── src/ │ └── main/ │ └── thrift/ │ ├── BUILD │ ├── ads.thrift │ ├── candidate_generation_key.thrift │ ├── cr_mixer.thrift │ ├── frs_based_tweet.thrift │ ├── metric_tags.thrift │ ├── product.thrift │ ├── product_context.thrift │ ├── related_tweet.thrift │ ├── related_video_tweet.thrift │ ├── scribe.thrift │ ├── source_type.thrift │ ├── topic_tweet.thrift │ ├── uteg.thrift │ └── validation.thrift ├── follow-recommendations-service/ │ ├── BUILD │ ├── CONFIG.ini │ ├── README.md │ ├── common/ │ │ └── src/ │ │ └── main/ │ │ └── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── follow_recommendations/ │ │ └── common/ │ │ ├── base/ │ │ │ ├── BUILD │ │ │ ├── CandidateSourceRegistry.scala │ │ │ ├── EnrichedCandidateSource.scala │ │ │ ├── ParamPredicate.scala │ │ │ ├── Predicate.scala │ │ │ ├── PredicateResult.scala │ │ │ ├── Ranker.scala │ │ │ ├── RecommendationFlow.scala │ │ │ ├── SideEffectsUtil.scala │ │ │ ├── StatsUtil.scala │ │ │ └── Transform.scala │ │ ├── candidate_sources/ │ │ │ ├── addressbook/ │ │ │ │ ├── AddressBookParams.scala │ │ │ │ ├── BUILD │ │ │ │ ├── ForwardEmailBookSource.scala │ │ │ │ ├── ForwardPhoneBookSource.scala │ │ │ │ ├── README.md │ │ │ │ ├── ReverseEmailBookSource.scala │ │ │ │ └── ReversePhoneBookSource.scala │ │ │ ├── base/ │ │ │ │ ├── BUILD │ │ │ │ ├── CachedCandidateSource.scala │ │ │ │ ├── ExperimentalCandidateSource.scala │ │ │ │ ├── RealGraphExpansionRepository.scala │ │ │ │ ├── SimilarUserExpanderParams.scala │ │ │ │ ├── SimilarUserExpanderRepository.scala │ │ │ │ ├── SocialProofEnforcedCandidateSource.scala │ │ │ │ ├── SocialProofEnforcedCandidateSourceFSConfig.scala │ │ │ │ ├── SocialProofEnforcedCandidateSourceParams.scala │ │ │ │ ├── StratoFetcherSource.scala │ │ │ │ ├── StratoFetcherWithUnitViewSource.scala │ │ │ │ ├── TweetAuthorsCandidateSource.scala │ │ │ │ └── TwoHopExpansionCandidateSource.scala │ │ │ ├── crowd_search_accounts/ │ │ │ │ ├── BUILD │ │ │ │ ├── CrowdSearchAccountsFSConfig.scala │ │ │ │ ├── CrowdSearchAccountsParams.scala │ │ │ │ ├── CrowdSearchAccountsSource.scala │ │ │ │ └── README.md │ │ │ ├── geo/ │ │ │ │ ├── BUILD │ │ │ │ ├── BasePopGeoHashSource.scala │ │ │ │ ├── PopCountryBackFillSource.scala │ │ │ │ ├── PopCountrySource.scala │ │ │ │ ├── PopGeoQualityFollowSource.scala │ │ │ │ ├── PopGeoQualityFollowSourceFSConfig.scala │ │ │ │ ├── PopGeoQualityFollowSourceParams.scala │ │ │ │ ├── PopGeoSource.scala │ │ │ │ ├── PopGeoSourceFSConfig.scala │ │ │ │ ├── PopGeoSourceParams.scala │ │ │ │ ├── PopGeohashSource.scala │ │ │ │ └── README.md │ │ │ ├── ppmi_locale_follow/ │ │ │ │ ├── BUILD │ │ │ │ ├── PPMILocaleFollowSource.scala │ │ │ │ ├── PPMILocaleFollowSourceFSConfig.scala │ │ │ │ ├── PPMILocaleFollowSourceParams.scala │ │ │ │ └── README.md │ │ │ ├── promoted_accounts/ │ │ │ │ ├── BUILD │ │ │ │ ├── PromotedAccountsCandidateSource.scala │ │ │ │ └── README.md │ │ │ ├── real_graph/ │ │ │ │ ├── BUILD │ │ │ │ ├── README.md │ │ │ │ ├── RealGraphOonFSConfig.scala │ │ │ │ ├── RealGraphOonParams.scala │ │ │ │ ├── RealGraphOonV2Source.scala │ │ │ │ └── RealGraphSource.scala │ │ │ ├── recent_engagement/ │ │ │ │ ├── BUILD │ │ │ │ ├── README.md │ │ │ │ ├── RecentEngagementDirectFollowSource.scala │ │ │ │ ├── RecentEngagementNonDirectFollowSource.scala │ │ │ │ ├── RepeatedProfileVisitsFSConfig.scala │ │ │ │ ├── RepeatedProfileVisitsParams.scala │ │ │ │ └── RepeatedProfileVisitsSource.scala │ │ │ ├── salsa/ │ │ │ │ ├── BUILD │ │ │ │ ├── README.md │ │ │ │ ├── RecentEngagementDirectFollowSalsaExpansionSource.scala │ │ │ │ ├── SalsaExpander.scala │ │ │ │ └── SalsaExpansionBasedCandidateSource.scala │ │ │ ├── sims/ │ │ │ │ ├── BUILD │ │ │ │ ├── CacheBasedSimsStore.scala │ │ │ │ ├── DBV2SimsRefreshStore.scala │ │ │ │ ├── DBV2SimsStore.scala │ │ │ │ ├── Follow2vecNearestNeighborsStore.scala │ │ │ │ ├── README.md │ │ │ │ ├── SimsExperimentalStore.scala │ │ │ │ ├── SimsSourceFSConfig.scala │ │ │ │ ├── SimsSourceParams.scala │ │ │ │ ├── SimsStore.scala │ │ │ │ ├── StratoBasedSimsCandidateSource.scala │ │ │ │ ├── StratoBasedSimsCandidateSourceWithUnitView.scala │ │ │ │ └── SwitchingSimsSource.scala │ │ │ ├── sims_expansion/ │ │ │ │ ├── BUILD │ │ │ │ ├── DBV2SimsExpansionParams.scala │ │ │ │ ├── README.md │ │ │ │ ├── RecentEngagementSimilarUsersFSConfig.scala │ │ │ │ ├── RecentEngagementSimilarUsersParams.scala │ │ │ │ ├── RecentEngagementSimilarUsersSource.scala │ │ │ │ ├── RecentFollowingSimilarUsersParams.scala │ │ │ │ ├── RecentFollowingSimilarUsersSource.scala │ │ │ │ ├── RecentStrongEngagementDirectFollowSimilarUsersSource.scala │ │ │ │ ├── SimsExpansionBasedCandidateSource.scala │ │ │ │ ├── SimsExpansionFSConfig.scala │ │ │ │ └── SimsExpansionSourceParams.scala │ │ │ ├── socialgraph/ │ │ │ │ ├── BUILD │ │ │ │ ├── README.md │ │ │ │ ├── RecentFollowingRecentFollowingExpansionSource.scala │ │ │ │ ├── RecentFollowingRecentFollowingExpansionSourceFSConfig.scala │ │ │ │ └── RecentFollowingRecentFollowingExpansionSourceParams.scala │ │ │ ├── stp/ │ │ │ │ ├── BUILD │ │ │ │ ├── BaseOnlineSTPSource.scala │ │ │ │ ├── Dbv2StpScorer.scala │ │ │ │ ├── EpStpScorer.scala │ │ │ │ ├── MutualFollowStrongTiePredictionSource.scala │ │ │ │ ├── OfflineMutualFollowExpansionSource.scala │ │ │ │ ├── OfflineStpSourceFsConfig.scala │ │ │ │ ├── OfflineStpSourceParams.scala │ │ │ │ ├── OfflineStpSourceWithDensePmiMatrix.scala │ │ │ │ ├── OfflineStpSourceWithLegacyPmiMatrix.scala │ │ │ │ ├── OfflineStrongTiePredictionBaseSource.scala │ │ │ │ ├── OfflineStrongTiePredictionSource.scala │ │ │ │ ├── OnlineSTPSourceFSConfig.scala │ │ │ │ ├── OnlineSTPSourceParams.scala │ │ │ │ ├── OnlineSTPSourceScorer.scala │ │ │ │ ├── OnlineSTPSourceWithDeepbirdV2Scorer.scala │ │ │ │ ├── OnlineSTPSourceWithEPScorer.scala │ │ │ │ ├── README.md │ │ │ │ ├── STPFirstDegreeFetcher.scala │ │ │ │ ├── STPGraphBuilder.scala │ │ │ │ ├── STPSecondDegreeFetcher.scala │ │ │ │ └── SocialProofEnforcedOfflineStrongTiePredictionSource.scala │ │ │ ├── top_organic_follows_accounts/ │ │ │ │ ├── BUILD │ │ │ │ ├── README.md │ │ │ │ ├── TopOrganicFollowsAccountsFSConfig.scala │ │ │ │ ├── TopOrganicFollowsAccountsParams.scala │ │ │ │ └── TopOrganicFollowsAccountsSource.scala │ │ │ ├── triangular_loops/ │ │ │ │ ├── BUILD │ │ │ │ ├── README.md │ │ │ │ ├── TriangularLoopsFSConfig.scala │ │ │ │ ├── TriangularLoopsParams.scala │ │ │ │ └── TriangularLoopsSource.scala │ │ │ ├── two_hop_random_walk/ │ │ │ │ ├── BUILD │ │ │ │ ├── README.md │ │ │ │ └── TwoHopRandomWalkSource.scala │ │ │ └── user_user_graph/ │ │ │ ├── BUILD │ │ │ ├── README.md │ │ │ ├── UserUserGraphCandidateSource.scala │ │ │ ├── UserUserGraphFSConfig.scala │ │ │ └── UserUserGraphParams.scala │ │ ├── clients/ │ │ │ ├── addressbook/ │ │ │ │ ├── AddressbookClient.scala │ │ │ │ ├── AddressbookModule.scala │ │ │ │ ├── BUILD │ │ │ │ └── models/ │ │ │ │ ├── BUILD │ │ │ │ ├── Contact.scala │ │ │ │ ├── EdgeType.scala │ │ │ │ ├── QueryOption.scala │ │ │ │ └── RecordIdentifier.scala │ │ │ ├── adserver/ │ │ │ │ ├── AdRequest.scala │ │ │ │ ├── AdserverClient.scala │ │ │ │ ├── AdserverModule.scala │ │ │ │ └── BUILD │ │ │ ├── cache/ │ │ │ │ ├── BUILD │ │ │ │ ├── MemcacheClient.scala │ │ │ │ ├── MemcacheModule.scala │ │ │ │ └── ThriftBijection.scala │ │ │ ├── common/ │ │ │ │ ├── BUILD │ │ │ │ └── BaseClientModule.scala │ │ │ ├── deepbirdv2/ │ │ │ │ ├── BUILD │ │ │ │ └── DeepBirdV2PredictionServiceClientModule.scala │ │ │ ├── dismiss_store/ │ │ │ │ ├── BUILD │ │ │ │ └── DismissStore.scala │ │ │ ├── email_storage_service/ │ │ │ │ ├── BUILD │ │ │ │ ├── EmailStorageServiceClient.scala │ │ │ │ └── EmailStorageServiceModule.scala │ │ │ ├── geoduck/ │ │ │ │ ├── BUILD │ │ │ │ ├── LocationServiceClient.scala │ │ │ │ ├── LocationServiceModule.scala │ │ │ │ ├── ReverseGeocodeClient.scala │ │ │ │ └── UserLocationFetcher.scala │ │ │ ├── gizmoduck/ │ │ │ │ ├── BUILD │ │ │ │ ├── GizmoduckClient.scala │ │ │ │ └── GizmoduckModule.scala │ │ │ ├── graph_feature_service/ │ │ │ │ ├── BUILD │ │ │ │ ├── GraphFeatureServiceClient.scala │ │ │ │ └── GraphFeatureStoreModule.scala │ │ │ ├── impression_store/ │ │ │ │ ├── BUILD │ │ │ │ ├── ImpressionStoreModule.scala │ │ │ │ └── WtfImpressionStore.scala │ │ │ ├── interests_service/ │ │ │ │ ├── BUILD │ │ │ │ └── InterestServiceClient.scala │ │ │ ├── phone_storage_service/ │ │ │ │ ├── BUILD │ │ │ │ ├── PhoneStorageServiceClient.scala │ │ │ │ └── PhoneStorageServiceModule.scala │ │ │ ├── real_time_real_graph/ │ │ │ │ ├── BUILD │ │ │ │ ├── Engagement.scala │ │ │ │ ├── EngagementScorer.scala │ │ │ │ └── RealTimeRealGraphClient.scala │ │ │ ├── socialgraph/ │ │ │ │ ├── BUILD │ │ │ │ ├── SocialGraphClient.scala │ │ │ │ └── SocialGraphModule.scala │ │ │ ├── strato/ │ │ │ │ ├── BUILD │ │ │ │ └── StratoClientModule.scala │ │ │ └── user_state/ │ │ │ ├── BUILD │ │ │ └── UserStateClient.scala │ │ ├── constants/ │ │ │ ├── BUILD │ │ │ ├── CandidateAlgorithmTypeConstants.scala │ │ │ ├── GuiceNamedConstants.scala │ │ │ └── ServiceConstants.scala │ │ ├── feature_hydration/ │ │ │ ├── adapters/ │ │ │ │ ├── BUILD │ │ │ │ ├── CandidateAlgorithmAdapter.scala │ │ │ │ ├── ClientContextAdapter.scala │ │ │ │ ├── PostNuxAlgorithmAdapter.scala │ │ │ │ └── PreFetchedFeatureAdapter.scala │ │ │ ├── common/ │ │ │ │ ├── BUILD │ │ │ │ ├── FeatureSource.scala │ │ │ │ ├── FeatureSourceId.scala │ │ │ │ └── HasPreFetchedFeature.scala │ │ │ └── sources/ │ │ │ ├── BUILD │ │ │ ├── CandidateAlgorithmSource.scala │ │ │ ├── ClientContextSource.scala │ │ │ ├── FeatureHydrationSourcesFSConfig.scala │ │ │ ├── FeatureHydrationSourcesFeatureSwitchKeys.scala │ │ │ ├── FeatureStoreFeatures.scala │ │ │ ├── FeatureStoreGizmoduckSource.scala │ │ │ ├── FeatureStoreParameters.scala │ │ │ ├── FeatureStorePostNuxAlgorithmSource.scala │ │ │ ├── FeatureStoreSource.scala │ │ │ ├── FeatureStoreSourceParams.scala │ │ │ ├── FeatureStoreTimelinesAuthorSource.scala │ │ │ ├── FeatureStoreUserMetricCountsSource.scala │ │ │ ├── HydrationSourcesModule.scala │ │ │ ├── PreFetchedFeatureSource.scala │ │ │ ├── UserScoringFeatureSource.scala │ │ │ └── Utils.scala │ │ ├── features/ │ │ │ ├── BUILD │ │ │ ├── LocationFeature.scala │ │ │ ├── TrackingTokenFeature.scala │ │ │ └── UserStateFeature.scala │ │ ├── models/ │ │ │ ├── AddressBookMetadata.scala │ │ │ ├── AlgorithmType.scala │ │ │ ├── BUILD │ │ │ ├── CandidateUser.scala │ │ │ ├── ClientContextConverter.scala │ │ │ ├── DisplayLocation.scala │ │ │ ├── EngagementType.scala │ │ │ ├── FilterReason.scala │ │ │ ├── FlowContext.scala │ │ │ ├── FlowRecommendation.scala │ │ │ ├── GeohashAndCountryCode.scala │ │ │ ├── HasAdMetadata.scala │ │ │ ├── HasByfSeedUserIds.scala │ │ │ ├── HasDataRecord.scala │ │ │ ├── HasDebugOptions.scala │ │ │ ├── HasDismissedUserIds.scala │ │ │ ├── HasDisplayLocation.scala │ │ │ ├── HasEngagements.scala │ │ │ ├── HasExcludedUserIds.scala │ │ │ ├── HasGeohashAndCountryCode.scala │ │ │ ├── HasInfoPerRankingStage.scala │ │ │ ├── HasInterestIds.scala │ │ │ ├── HasInvalidRelationshipUserIds.scala │ │ │ ├── HasIsSoftUser.scala │ │ │ ├── HasMutualFollowedUserIds.scala │ │ │ ├── HasPreviousRecommendationsContext.scala │ │ │ ├── HasProfileId.scala │ │ │ ├── HasQualityFactor.scala │ │ │ ├── HasRecentFollowedByUserIds.scala │ │ │ ├── HasRecentFollowedUserIds.scala │ │ │ ├── HasRecentFollowedUserIdsWithTime.scala │ │ │ ├── HasRecentlyEngagedUserIds.scala │ │ │ ├── HasRecommendationFlowIdentifier.scala │ │ │ ├── HasScores.scala │ │ │ ├── HasSimilarToContext.scala │ │ │ ├── HasTopicId.scala │ │ │ ├── HasUserCandidateSourceDetails.scala │ │ │ ├── HasUserState.scala │ │ │ ├── HasWtfImpressions.scala │ │ │ ├── OptimusRequest.scala │ │ │ ├── Product.scala │ │ │ ├── RankingInfo.scala │ │ │ ├── Reason.scala │ │ │ ├── RecentlyEngagedUserId.scala │ │ │ ├── RecommendationStep.scala │ │ │ ├── STPGraph.scala │ │ │ ├── SafetyLevel.scala │ │ │ ├── Score.scala │ │ │ ├── Session.scala │ │ │ ├── SignalData.scala │ │ │ ├── TrackingToken.scala │ │ │ ├── TweetCandidate.scala │ │ │ ├── UserCandidateSourceDetails.scala │ │ │ ├── UserIdAndTimestamp.scala │ │ │ └── WtfImpression.scala │ │ ├── predicates/ │ │ │ ├── BUILD │ │ │ ├── CandidateParamPredicate.scala │ │ │ ├── CandidateSourceParamPredicate.scala │ │ │ ├── CuratedCompetitorListPredicate.scala │ │ │ ├── ExcludedUserIdPredicate.scala │ │ │ ├── InactivePredicate.scala │ │ │ ├── InactivePredicateParams.scala │ │ │ ├── PreviouslyRecommendedUserIdsPredicate.scala │ │ │ ├── dismiss/ │ │ │ │ ├── BUILD │ │ │ │ ├── DismissedCandidatePredicate.scala │ │ │ │ └── DismissedCandidatePredicateParams.scala │ │ │ ├── gizmoduck/ │ │ │ │ ├── BUILD │ │ │ │ ├── GizmoduckPredicate.scala │ │ │ │ ├── GizmoduckPredicateCache.scala │ │ │ │ ├── GizmoduckPredicateFSConfig.scala │ │ │ │ └── GizmoduckPredicateParams.scala │ │ │ ├── health/ │ │ │ │ ├── BUILD │ │ │ │ ├── HssPredicate.scala │ │ │ │ ├── HssPredicateFSConfig.scala │ │ │ │ └── HssPredicateParams.scala │ │ │ ├── sgs/ │ │ │ │ ├── BUILD │ │ │ │ ├── InvalidRelationshipPredicate.scala │ │ │ │ ├── RecentFollowingPredicate.scala │ │ │ │ ├── SgsPredicateFSConfig.scala │ │ │ │ ├── SgsPredicateParams.scala │ │ │ │ ├── SgsRelationshipsByUserIdPredicate.scala │ │ │ │ └── SgsRelationshipsPredicate.scala │ │ │ └── user_activity/ │ │ │ ├── BUILD │ │ │ ├── UserActivityPredicate.scala │ │ │ └── UserActivityPredicateParams.scala │ │ ├── rankers/ │ │ │ ├── common/ │ │ │ │ ├── AdhocScoreModificationType.scala │ │ │ │ ├── BUILD │ │ │ │ ├── DedupCandidates.scala │ │ │ │ └── RankerId.scala │ │ │ ├── fatigue_ranker/ │ │ │ │ ├── BUILD │ │ │ │ ├── ImpressionBasedFatigueRanker.scala │ │ │ │ ├── ImpressionBasedFatigueRankerFSConfig.scala │ │ │ │ └── ImpressionBasedFatigueRankerParams.scala │ │ │ ├── first_n_ranker/ │ │ │ │ ├── BUILD │ │ │ │ ├── FirstNRanker.scala │ │ │ │ ├── FirstNRankerFSConfig.scala │ │ │ │ ├── FirstNRankerFeatureSwitchKeys.scala │ │ │ │ └── FirstNRankerParams.scala │ │ │ ├── interleave_ranker/ │ │ │ │ ├── BUILD │ │ │ │ ├── InterleaveRanker.scala │ │ │ │ ├── InterleaveRankerFSConfig.scala │ │ │ │ └── InterleaveRankerParams.scala │ │ │ ├── ml_ranker/ │ │ │ │ ├── ranking/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── HydrateFeaturesTransform.scala │ │ │ │ │ ├── MlRanker.scala │ │ │ │ │ ├── MlRankerFSConfig.scala │ │ │ │ │ └── MlRankerParams.scala │ │ │ │ └── scoring/ │ │ │ │ ├── AdhocScorer.scala │ │ │ │ ├── BUILD │ │ │ │ ├── DeepbirdScorer.scala │ │ │ │ ├── PostnuxDeepbirdProdScorer.scala │ │ │ │ ├── RandomScorer.scala │ │ │ │ ├── Scorer.scala │ │ │ │ └── ScorerFactory.scala │ │ │ ├── utils/ │ │ │ │ ├── BUILD │ │ │ │ └── Utils.scala │ │ │ └── weighted_candidate_source_ranker/ │ │ │ ├── BUILD │ │ │ ├── CandidateShuffle.scala │ │ │ ├── WeightMethod.scala │ │ │ ├── WeightedCandidateSourceBaseRanker.scala │ │ │ ├── WeightedCandidateSourceRanker.scala │ │ │ ├── WeightedCandidateSourceRankerFSConfig.scala │ │ │ └── WeightedCandidateSourceRankerParams.scala │ │ ├── stores/ │ │ │ ├── BUILD │ │ │ └── LowTweepCredFollowStore.scala │ │ ├── transforms/ │ │ │ ├── dedup/ │ │ │ │ ├── BUILD │ │ │ │ └── DedupTransform.scala │ │ │ ├── modify_social_proof/ │ │ │ │ ├── BUILD │ │ │ │ ├── ModifySocialProofTransform.scala │ │ │ │ └── RemoveAccountProofTransform.scala │ │ │ ├── ranker_id/ │ │ │ │ ├── BUILD │ │ │ │ └── RandomRankerIdTransform.scala │ │ │ ├── recommendation_flow_identifier/ │ │ │ │ ├── AddRecommendationFlowIdentifierTransform.scala │ │ │ │ └── BUILD │ │ │ ├── tracking_token/ │ │ │ │ ├── BUILD │ │ │ │ └── TrackingTokenTransform.scala │ │ │ └── weighted_sampling/ │ │ │ ├── BUILD │ │ │ ├── SamplingTransform.scala │ │ │ ├── SamplingTransformFSConfig.scala │ │ │ └── SamplingTransformParams.scala │ │ └── utils/ │ │ ├── BUILD │ │ ├── CollectionUtil.scala │ │ ├── DisplayLocationProductConverterUtil.scala │ │ ├── MergeUtil.scala │ │ ├── RandomUtil.scala │ │ ├── RescueWithStatsUtils.scala │ │ ├── UserSignupUtil.scala │ │ └── Weighted.scala │ ├── server/ │ │ └── src/ │ │ └── main/ │ │ ├── resources/ │ │ │ ├── BUILD │ │ │ ├── config/ │ │ │ │ └── decider.yml │ │ │ ├── logback.xml │ │ │ └── quality/ │ │ │ └── stp_models/ │ │ │ └── 20141223/ │ │ │ ├── epModel │ │ │ └── trainingConfig │ │ └── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── follow_recommendations/ │ │ ├── BUILD │ │ ├── FollowRecommendationsServiceThriftServer.scala │ │ ├── assembler/ │ │ │ └── models/ │ │ │ ├── Action.scala │ │ │ ├── BUILD │ │ │ ├── Config.scala │ │ │ ├── FeedbackAction.scala │ │ │ ├── Footer.scala │ │ │ ├── Header.scala │ │ │ ├── Layout.scala │ │ │ ├── RecommendationOptions.scala │ │ │ ├── SocialProof.scala │ │ │ ├── Title.scala │ │ │ └── WTFPresentation.scala │ │ ├── blenders/ │ │ │ ├── BUILD │ │ │ └── PromotedAccountsBlender.scala │ │ ├── configapi/ │ │ │ ├── BUILD │ │ │ ├── ConfigBuilder.scala │ │ │ ├── DeciderConfigs.scala │ │ │ ├── FeatureSwitchConfigs.scala │ │ │ ├── GlobalFeatureSwitchConfig.scala │ │ │ ├── ParamsFactory.scala │ │ │ ├── RequestContext.scala │ │ │ ├── RequestContextFactory.scala │ │ │ ├── candidates/ │ │ │ │ ├── BUILD │ │ │ │ ├── CandidateUserContext.scala │ │ │ │ ├── CandidateUserContextFactory.scala │ │ │ │ ├── CandidateUserParamsFactory.scala │ │ │ │ └── HydrateCandidateParamsTransform.scala │ │ │ ├── common/ │ │ │ │ ├── BUILD │ │ │ │ └── FeatureSwitchConfig.scala │ │ │ ├── deciders/ │ │ │ │ ├── BUILD │ │ │ │ ├── DeciderKey.scala │ │ │ │ └── DeciderParams.scala │ │ │ └── params/ │ │ │ ├── BUILD │ │ │ └── GlobalParams.scala │ │ ├── controllers/ │ │ │ ├── BUILD │ │ │ ├── CandidateUserDebugParamsBuilder.scala │ │ │ ├── RecommendationRequestBuilder.scala │ │ │ ├── RequestBuilderUserFetcher.scala │ │ │ ├── ScoringUserRequestBuilder.scala │ │ │ └── ThriftController.scala │ │ ├── flows/ │ │ │ ├── ads/ │ │ │ │ ├── BUILD │ │ │ │ ├── PromotedAccountsFlow.scala │ │ │ │ ├── PromotedAccountsFlowParams.scala │ │ │ │ ├── PromotedAccountsFlowRequest.scala │ │ │ │ └── PromotedAccountsUtil.scala │ │ │ ├── content_recommender_flow/ │ │ │ │ ├── BUILD │ │ │ │ ├── ContentRecommenderFlow.scala │ │ │ │ ├── ContentRecommenderFlowCandidateSourceRegistry.scala │ │ │ │ ├── ContentRecommenderFlowCandidateSourceWeights.scala │ │ │ │ ├── ContentRecommenderFlowCandidateSourceWeightsParams.scala │ │ │ │ ├── ContentRecommenderFlowFSConfig.scala │ │ │ │ ├── ContentRecommenderFlowFeatureSwitchKeys.scala │ │ │ │ ├── ContentRecommenderParams.scala │ │ │ │ ├── ContentRecommenderRequest.scala │ │ │ │ └── ContentRecommenderRequestBuilder.scala │ │ │ └── post_nux_ml/ │ │ │ ├── BUILD │ │ │ ├── PostNuxMlCandidateSourceRegistry.scala │ │ │ ├── PostNuxMlCandidateSourceWeightParams.scala │ │ │ ├── PostNuxMlCombinedRankerBuilder.scala │ │ │ ├── PostNuxMlFlow.scala │ │ │ ├── PostNuxMlFlowCandidateSourceWeights.scala │ │ │ ├── PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.scala │ │ │ ├── PostNuxMlFlowFSConfig.scala │ │ │ ├── PostNuxMlFlowFeatureSwitchKeys.scala │ │ │ ├── PostNuxMlParams.scala │ │ │ ├── PostNuxMlRequest.scala │ │ │ ├── PostNuxMlRequestBuilder.scala │ │ │ └── PostNuxMlRequestBuilderParams.scala │ │ ├── logging/ │ │ │ ├── BUILD │ │ │ └── FrsLogger.scala │ │ ├── models/ │ │ │ ├── BUILD │ │ │ ├── CandidateSourceType.scala │ │ │ ├── CandidateUserDebugParams.scala │ │ │ ├── DebugParams.scala │ │ │ ├── DisplayContext.scala │ │ │ ├── FeatureValue.scala │ │ │ ├── RecommendationFlowData.scala │ │ │ ├── RecommendationRequest.scala │ │ │ ├── RecommendationResponse.scala │ │ │ ├── Request.scala │ │ │ ├── ScoringUserRequest.scala │ │ │ ├── ScoringUserResponse.scala │ │ │ └── failures/ │ │ │ ├── BUILD │ │ │ └── TimeoutPipelineFailure.scala │ │ ├── modules/ │ │ │ ├── ABDeciderModule.scala │ │ │ ├── BUILD │ │ │ ├── ConfigApiModule.scala │ │ │ ├── DiffyModule.scala │ │ │ ├── FeatureSwitchesModule.scala │ │ │ ├── FlagsModule.scala │ │ │ ├── ProductRegistryModule.scala │ │ │ ├── ScorerModule.scala │ │ │ ├── ScribeModule.scala │ │ │ └── TimerModule.scala │ │ ├── products/ │ │ │ ├── BUILD │ │ │ ├── ProdProductRegistry.scala │ │ │ ├── common/ │ │ │ │ ├── BUILD │ │ │ │ ├── Exceptions.scala │ │ │ │ ├── Product.scala │ │ │ │ └── ProductRegistry.scala │ │ │ ├── explore_tab/ │ │ │ │ ├── BUILD │ │ │ │ ├── ExploreTabProduct.scala │ │ │ │ └── configapi/ │ │ │ │ ├── BUILD │ │ │ │ ├── ExploreTabFSConfig.scala │ │ │ │ └── ExploreTabParams.scala │ │ │ ├── home_timeline/ │ │ │ │ ├── BUILD │ │ │ │ ├── HTLProductMixer.scala │ │ │ │ ├── HomeTimelineProduct.scala │ │ │ │ ├── HomeTimelineStrings.scala │ │ │ │ └── configapi/ │ │ │ │ ├── BUILD │ │ │ │ ├── HomeTimelineFSConfig.scala │ │ │ │ └── HomeTimelineParams.scala │ │ │ ├── home_timeline_tweet_recs/ │ │ │ │ ├── BUILD │ │ │ │ ├── HomeTimelineTweetRecsProduct.scala │ │ │ │ └── configapi/ │ │ │ │ ├── BUILD │ │ │ │ └── HomeTimelineTweetRecsParams.scala │ │ │ └── sidebar/ │ │ │ ├── BUILD │ │ │ ├── SidebarProduct.scala │ │ │ └── configapi/ │ │ │ ├── BUILD │ │ │ └── SidebarParams.scala │ │ ├── services/ │ │ │ ├── BUILD │ │ │ ├── FollowRecommendationsServiceWarmupHandler.scala │ │ │ ├── ProductMixerRecommendationService.scala │ │ │ ├── ProductPipelineSelector.scala │ │ │ ├── ProductPipelineSelectorConfig.scala │ │ │ ├── ProductRecommenderService.scala │ │ │ ├── RecommendationsService.scala │ │ │ ├── UserScoringService.scala │ │ │ └── exceptions/ │ │ │ ├── BUILD │ │ │ └── UnknownExceptionMapper.scala │ │ └── utils/ │ │ ├── BUILD │ │ ├── CandidateSourceHoldbackUtil.scala │ │ └── RecommendationFlowBaseSideEffectsUtil.scala │ └── thrift/ │ └── src/ │ └── main/ │ └── thrift/ │ ├── BUILD │ ├── assembler.thrift │ ├── client_context.thrift │ ├── debug.thrift │ ├── display_context.thrift │ ├── display_location.thrift │ ├── engagementType.thrift │ ├── flows.thrift │ ├── follow-recommendations-service.thrift │ ├── follow_recommendations_serving_history.thrift │ ├── logging/ │ │ ├── BUILD │ │ ├── client_context.thrift │ │ ├── debug.thrift │ │ ├── display_context.thrift │ │ ├── display_location.thrift │ │ ├── engagementType.thrift │ │ ├── flows.thrift │ │ ├── logs.thrift │ │ ├── reasons.thrift │ │ ├── recently_engaged_user_id.thrift │ │ ├── recommendations.thrift │ │ ├── scoring.thrift │ │ └── tracking.thrift │ ├── reasons.thrift │ ├── recently_engaged_user_id.thrift │ ├── recommendations.thrift │ ├── scoring.thrift │ └── tracking.thrift ├── graph-feature-service/ │ ├── BUILD.bazel │ ├── README.md │ ├── doc/ │ │ ├── common.md │ │ └── getintersection.md │ └── src/ │ └── main/ │ ├── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── graph_feature_service/ │ │ ├── common/ │ │ │ ├── BUILD.bazel │ │ │ └── Configs.scala │ │ ├── server/ │ │ │ ├── BUILD.bazel │ │ │ ├── Main.scala │ │ │ ├── controllers/ │ │ │ │ └── ServerController.scala │ │ │ ├── handlers/ │ │ │ │ ├── ServerGetIntersectionHandler.scala │ │ │ │ └── ServerWarmupHandler.scala │ │ │ ├── modules/ │ │ │ │ ├── GetIntersectionStoreModule.scala │ │ │ │ ├── GraphFeatureServiceWorkerClientsModule.scala │ │ │ │ ├── LZ4Injection.scala │ │ │ │ └── ServerFlagModule.scala │ │ │ └── stores/ │ │ │ ├── FeatureTypesEncoder.scala │ │ │ └── GetIntersectionStore.scala │ │ ├── util/ │ │ │ ├── BUILD │ │ │ ├── FeatureTypesCalculator.scala │ │ │ └── IntersectionValueCalculator.scala │ │ └── worker/ │ │ ├── BUILD.bazel │ │ ├── Main.scala │ │ ├── controllers/ │ │ │ └── WorkerController.scala │ │ ├── handlers/ │ │ │ ├── WorkerGetIntersectionHandler.scala │ │ │ └── WorkerWarmupHandler.scala │ │ ├── modules/ │ │ │ ├── GraphContainerProviderModule.scala │ │ │ └── WorkerFlagModule.scala │ │ └── util/ │ │ ├── AutoUpdatingGraph.scala │ │ ├── GfsQuery.scala │ │ ├── GraphContainer.scala │ │ ├── GraphKey.scala │ │ └── GraphType.scala │ ├── scalding/ │ │ └── com/ │ │ └── twitter/ │ │ └── graph_feature_service/ │ │ └── scalding/ │ │ ├── BUILD.bazel │ │ ├── EdgeFeature.scala │ │ ├── GraphFeatureServiceAppBase.scala │ │ ├── GraphFeatureServiceApps.scala │ │ ├── GraphFeatureServiceMainJob.scala │ │ └── adhoc/ │ │ ├── BUILD.bazel │ │ └── RandomRequestGenerationApp.scala │ └── thrift/ │ └── com/ │ └── twitter/ │ └── graph_feature_service/ │ ├── BUILD │ └── graph_feature_service.thrift ├── home-mixer/ │ ├── BUILD.bazel │ ├── README.md │ └── server/ │ └── src/ │ └── main/ │ └── scala/ │ └── com/ │ └── twitter/ │ └── home_mixer/ │ ├── BUILD.bazel │ ├── HomeMixerHttpServerWarmupHandler.scala │ ├── HomeMixerServer.scala │ ├── HomeMixerThriftServerWarmupHandler.scala │ ├── candidate_pipeline/ │ │ ├── BUILD.bazel │ │ ├── ConversationServiceCandidatePipelineConfig.scala │ │ ├── ConversationServiceCandidatePipelineConfigBuilder.scala │ │ ├── ConversationServiceResponseFeatureTransformer.scala │ │ ├── EditedTweetsCandidatePipelineConfig.scala │ │ ├── NewTweetsPillCandidatePipelineConfig.scala │ │ ├── TimelineServiceResponseFeatureTransformer.scala │ │ └── VerifiedPromptCandidatePipelineConfig.scala │ ├── controller/ │ │ ├── BUILD.bazel │ │ ├── HomeHttpController.scala │ │ └── HomeThriftController.scala │ ├── federated/ │ │ ├── BUILD.bazel │ │ └── HomeMixerColumn.scala │ ├── functional_component/ │ │ ├── candidate_source/ │ │ │ ├── BUILD.bazel │ │ │ ├── EarlybirdCandidateSource.scala │ │ │ └── StaleTweetsCacheCandidateSource.scala │ │ ├── decorator/ │ │ │ ├── BUILD.bazel │ │ │ ├── EntryPointPivotModuleDecorator.scala │ │ │ ├── ForYouTweetCandidateDecorator.scala │ │ │ ├── HomeConversationServiceCandidateDecorator.scala │ │ │ ├── HomeQueryTypePredicates.scala │ │ │ ├── KeywordTrendsModuleCandidateDecorator.scala │ │ │ ├── PinnedTweetBroadcastCandidateDecorator.scala │ │ │ ├── StoriesModuleCandidateDecorator.scala │ │ │ ├── TuneFeedModuleCandidateDecorator.scala │ │ │ ├── TweetCarouselModuleCandidateDecorator.scala │ │ │ ├── VideoCarouselModuleCandidateDecorator.scala │ │ │ ├── builder/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── HomeAdsClientEventDetailsBuilder.scala │ │ │ │ ├── HomeClientEventDetailsBuilder.scala │ │ │ │ ├── HomeClientEventInfoBuilder.scala │ │ │ │ ├── HomeConversationModuleMetadataBuilder.scala │ │ │ │ ├── HomeTimelinesScoreInfoBuilder.scala │ │ │ │ ├── HomeTweetTypePredicates.scala │ │ │ │ ├── KeywordTrendMetaDescriptionBuilder.scala │ │ │ │ ├── ListClientEventDetailsBuilder.scala │ │ │ │ └── VerifiedPromptBuilder.scala │ │ │ └── urt/ │ │ │ └── builder/ │ │ │ ├── AddEntriesWithReplaceAndShowAlertAndShowCoverInstructionBuilder.scala │ │ │ ├── AuthorChildFeedbackActionBuilder.scala │ │ │ ├── BUILD.bazel │ │ │ ├── BlockUserChildFeedbackActionBuilder.scala │ │ │ ├── ChildFeedbackActionBuilder.scala │ │ │ ├── DebugSocialContextBuilder.scala │ │ │ ├── DontLikeFeedbackActionBuilder.scala │ │ │ ├── EngagerSocialContextBuilder.scala │ │ │ ├── ExtendedReplySocialContextBuilder.scala │ │ │ ├── FeedbackStrings.scala │ │ │ ├── FeedbackUtil.scala │ │ │ ├── FollowedBySocialContextBuilder.scala │ │ │ ├── HomeFeedbackActionInfoBuilder.scala │ │ │ ├── HomeTweetContextBuilder.scala │ │ │ ├── HomeTweetSocialContextBuilder.scala │ │ │ ├── HomeWhoToFollowFeedbackActionInfoBuilder.scala │ │ │ ├── HomeWhoToSubscribeFeedbackActionInfoBuilder.scala │ │ │ ├── LikedBySocialContextBuilder.scala │ │ │ ├── ListsSocialContextBuilder.scala │ │ │ ├── MuteUserChildFeedbackActionBuilder.scala │ │ │ ├── NotInterestedTopicFeedbackActionBuilder.scala │ │ │ ├── NotRelevantChildFeedbackActionBuilder.scala │ │ │ ├── PopularInYourAreaSocialContextBuilder.scala │ │ │ ├── PopularVideoSocialContextBuilder.scala │ │ │ ├── PostDetailsNegativeFeedbackActionBuilder.scala │ │ │ ├── PostFeedbackActionBuilder.scala │ │ │ ├── PostFollowupFeedbackActionBuilder.scala │ │ │ ├── ReceivedReplySocialContextBuilder.scala │ │ │ ├── RelevancePromptCandidateUrtItemBuilder.scala │ │ │ ├── ReportTweetChildFeedbackActionBuilder.scala │ │ │ ├── RetweeterChildFeedbackActionBuilder.scala │ │ │ ├── ServedTypeSocialContextBuilder.scala │ │ │ ├── TopicSocialContextBuilder.scala │ │ │ ├── TuneFeedFeedbackActionInfoBuilder.scala │ │ │ └── UnfollowUserChildFeedbackActionBuilder.scala │ │ ├── feature_hydrator/ │ │ │ ├── AncestorFeatureHydrator.scala │ │ │ ├── AuthorFeatureHydrator.scala │ │ │ ├── AuthorLargeEmbeddingsFeatureHydrator.scala │ │ │ ├── BUILD.bazel │ │ │ ├── BasketballContextFeatureHydrator.scala │ │ │ ├── BroadcastStateFeatureHydrator.scala │ │ │ ├── CategoryDiversityRescoringFeatureHydrator.scala │ │ │ ├── ClipEmbeddingFeatureHydrator.scala │ │ │ ├── ClipEmbeddingMediaUnderstandingFeatureHydrator.scala │ │ │ ├── ClipImageClusterIdFeatureHydrator.scala │ │ │ ├── DependentBulkCandidateFeatureHydrator.scala │ │ │ ├── DismissInfoQueryFeatureHydrator.scala │ │ │ ├── DiversityRescoringFeatureHydrator.scala │ │ │ ├── EarlybirdSearchResultFeatureHydrator.scala │ │ │ ├── FeedbackHistoryQueryFeatureHydrator.scala │ │ │ ├── FollowableUttTopicsQueryFeatureHydrator.scala │ │ │ ├── FrsSeedUsersQueryFeatureHydrator.scala │ │ │ ├── GeoduckAuthorLocationHydrator.scala │ │ │ ├── GizmoduckAuthorFeatureHydrator.scala │ │ │ ├── GizmoduckUserQueryFeatureHydrator.scala │ │ │ ├── GraphTwoHopFeatureHydrator.scala │ │ │ ├── GrokAnnotationsFeatureHydrator.scala │ │ │ ├── GrokGorkContentCreatorFeatureHydrator.scala │ │ │ ├── GrokTranslatedPostIsCachedFeatureHydrator.scala │ │ │ ├── HeartbeatOptimizerParamsHydrator.scala │ │ │ ├── HeavyRankerWeightsQueryFeatureHydrator.scala │ │ │ ├── ImpressedImageClusterIdsQueryFeatureHydrator.scala │ │ │ ├── ImpressedMediaClusterIdsQueryFeatureHydrator.scala │ │ │ ├── ImpressionBloomFilterQueryFeatureHydrator.scala │ │ │ ├── InNetworkFeatureHydrator.scala │ │ │ ├── LastNegativeFeedbackTimeQueryFeatureHydrator.scala │ │ │ ├── LastNonPollingTimeQueryFeatureHydrator.scala │ │ │ ├── ListIdsQueryFeatureHydrator.scala │ │ │ ├── MediaClusterIdFeatureHydrator.scala │ │ │ ├── MediaCompletionRateFeatureHydrator.scala │ │ │ ├── MultiModalEmbeddingsFeatureHydrator.scala │ │ │ ├── NamesFeatureHydrator.scala │ │ │ ├── NaviClientConfigQueryFeatureHydrator.scala │ │ │ ├── NaviVideoClientConfigQueryFeatureHydrator.scala │ │ │ ├── OnPremRealGraphQueryFeatureHydrator.scala │ │ │ ├── OptimizerWeightsQueryFeatureHydrator.scala │ │ │ ├── OriginalAuthorLargeEmbeddingsFeatureHydrator.scala │ │ │ ├── OriginalTweetLargeEmbeddingsFeatureHydrator.scala │ │ │ ├── PersistenceStoreQueryFeatureHydrator.scala │ │ │ ├── PerspectiveFilteredSocialContextFeatureHydrator.scala │ │ │ ├── PhoenixRescoringFeatureHydrator.scala │ │ │ ├── PostContextFeatureHydrator.scala │ │ │ ├── RateLimitQueryFeatureHydrator.scala │ │ │ ├── RealGraphInNetworkScoresQueryFeatureHydrator.scala │ │ │ ├── RealGraphQueryFeatureHydrator.scala │ │ │ ├── RealGraphViewerAuthorFeatureHydrator.scala │ │ │ ├── RealGraphViewerRelatedUsersFeatureHydrator.scala │ │ │ ├── RealTimeEntityRealGraphQueryFeatureHydrator.scala │ │ │ ├── RealTimeInteractionGraphEdgeFeatureHydrator.scala │ │ │ ├── RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala │ │ │ ├── RequestQueryFeatureHydrator.scala │ │ │ ├── RequestTimeQueryFeatureHydrator.scala │ │ │ ├── SGSValidSocialContextFeatureHydrator.scala │ │ │ ├── SimClustersEngagementSimilarityFeatureHydrator.scala │ │ │ ├── SimClustersLogFavBasedTweetFeatureHydrator.scala │ │ │ ├── SimClustersUserSparseEmbeddingsQueryFeatureHydrator.scala │ │ │ ├── SimClustersUserTweetScoresHydrator.scala │ │ │ ├── SimclusterBasedTopAuthorsQueryFeatureHydrator.scala │ │ │ ├── SlopAuthorFeatureHydrator.scala │ │ │ ├── SpaceStateFeatureHydrator.scala │ │ │ ├── TSPInferredTopicFeatureHydrator.scala │ │ │ ├── TransformerPostEmbeddingFeatureHydrator.scala │ │ │ ├── TweetEntityServiceContentFeatureHydrator.scala │ │ │ ├── TweetEntityServiceFeatureHydrator.scala │ │ │ ├── TweetImpressionsQueryFeatureHydrator.scala │ │ │ ├── TweetLanguageFeatureHydrator.scala │ │ │ ├── TweetLargeEmbeddingsFeatureHydrator.scala │ │ │ ├── TweetMetaDataFeatureHydrator.scala │ │ │ ├── TweetTimeFeatureHydrator.scala │ │ │ ├── TweetTypeMetricsFeatureHydrator.scala │ │ │ ├── TweetypieFeatureHydrator.scala │ │ │ ├── TwhinAuthorFollowFeatureHydrator.scala │ │ │ ├── TwhinRebuildTweetFeatureHydrator.scala │ │ │ ├── TwhinRebuildUserEngagementQueryFeatureHydrator.scala │ │ │ ├── TwhinRebuildUserPositiveQueryFeatureHydrator.scala │ │ │ ├── TwhinTweetFeatureHydrator.scala │ │ │ ├── TwhinUserEngagementQueryFeatureHydrator.scala │ │ │ ├── TwhinUserFollowQueryFeatureHydrator.scala │ │ │ ├── TwhinUserNegativeFeatureHydrator.scala │ │ │ ├── TwhinUserPositiveFeatureHydrator.scala │ │ │ ├── TwhinVideoFeatureHydrator.scala │ │ │ ├── UnifiedUserActionsUserIdentifierFeatureHydrator.scala │ │ │ ├── UserActionByteArrayQueryFeatureHydrator.scala │ │ │ ├── UserActionsQueryFeatureHydrator.scala │ │ │ ├── UserEngagedGrokCategoriesFeatureHydrator.scala │ │ │ ├── UserEngagedLanguagesFeatureHydrator.scala │ │ │ ├── UserEngagementGrokTagFeatureHydrator.scala │ │ │ ├── UserFrequentLocationHydrator.scala │ │ │ ├── UserFrequentLocationQueryFeatureHydrator.scala │ │ │ ├── UserHistoryTransformerEmbeddingQueryFeatureHydrator.scala │ │ │ ├── UserLanguagesFeatureHydrator.scala │ │ │ ├── UserLargeEmbeddingsFeatureHydrator.scala │ │ │ ├── UserStateQueryFeatureHydrator.scala │ │ │ ├── UserSubscriptionQueryFeatureHydrator.scala │ │ │ ├── UserUnderstandableLangaugesFeatureHydrator.scala │ │ │ ├── UtegFeatureHydrator.scala │ │ │ ├── VideoSummaryEmbeddingFeatureHydrator.scala │ │ │ ├── ViewCountsFeatureHydrator.scala │ │ │ ├── ViralContentCreatorMetricsFeatureHydrator.scala │ │ │ ├── WithDefaultFeatureMap.scala │ │ │ ├── adapters/ │ │ │ │ ├── author_features/ │ │ │ │ │ ├── AuthorFeaturesAdapter.scala │ │ │ │ │ └── BUILD.bazel │ │ │ │ ├── content/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ ├── ClipEmbeddingFeaturesAdapter.scala │ │ │ │ │ ├── ContentFeatureAdapter.scala │ │ │ │ │ ├── TextTokensFeaturesAdapter.scala │ │ │ │ │ └── VideoSummaryEmbeddingFeaturesAdaptor.scala │ │ │ │ ├── gizmoduck_features/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ └── GizmoduckFeaturesAdapter.scala │ │ │ │ ├── inferred_topic/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ └── InferredTopicAdapter.scala │ │ │ │ ├── light_ranking_features/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ └── LightRankingCandidateFeaturesAdapter.scala │ │ │ │ ├── non_ml_features/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ ├── NonMLCandidateFeaturesAdapter.scala │ │ │ │ │ └── NonMLCommonFeaturesAdapter.scala │ │ │ │ ├── offline_aggregates/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ ├── PassThroughAdapter.scala │ │ │ │ │ └── SparseAggregatesToDenseAdapter.scala │ │ │ │ ├── simclusters_features/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ └── SimclustersFeaturesAdapter.scala │ │ │ │ ├── transformer_embeddings/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ ├── TransformerEmbeddingsAdapter.scala │ │ │ │ │ ├── UserHistoryEventsAdapter.scala │ │ │ │ │ └── VideoUserHistoryEventsAdapter.scala │ │ │ │ └── twhin_embeddings/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── TwhinEmbeddingsAdapter.scala │ │ │ ├── offline_aggregates/ │ │ │ │ ├── AggregateFeatureInfo.scala │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── BaseEdgeAggregateFeatureHydrator.scala │ │ │ │ ├── EdgeAggregateFeatures.scala │ │ │ │ ├── PartAAggregateQueryFeatureHydrator.scala │ │ │ │ ├── PartBAggregateQueryFeatureHydrator.scala │ │ │ │ ├── TopicEdgeAggregateFeatureHydrator.scala │ │ │ │ ├── TopicEdgeTruncatedAggregateFeatureHydrator.scala │ │ │ │ ├── TweetContentEdgeAggregateFeatureHydrator.scala │ │ │ │ ├── UserEngagerEdgeAggregateFeatureHydrator.scala │ │ │ │ ├── UserEntityEdgeAggregateFeatureHydrator.scala │ │ │ │ └── Utils.scala │ │ │ ├── real_time_aggregates/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala │ │ │ │ ├── FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator.scala │ │ │ │ ├── TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala │ │ │ │ ├── TopicEngagementRealTimeAggregateFeatureHydrator.scala │ │ │ │ ├── TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala │ │ │ │ ├── TweetEngagementRealTimeAggregateFeatureHydrator.scala │ │ │ │ ├── TwitterListEngagementRealTimeAggregateFeatureHydrator.scala │ │ │ │ ├── UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala │ │ │ │ ├── UserEngagementRealTimeAggregatesFeatureHydrator.scala │ │ │ │ └── UserTweetTvVideoRealTimeAggregateFeatureHydrator.scala │ │ │ └── user_history/ │ │ │ ├── BUILD.bazel │ │ │ ├── BaseUserHistoryEventsQueryFeatureHydrator.scala │ │ │ ├── ScoredTweetsUserHistoryEventsQueryFeatureHydrator.scala │ │ │ └── ScoredVideoTweetsUserHistoryEventsQueryFeatureHydrator.scala │ │ ├── filter/ │ │ │ ├── AuthorDedupFilter.scala │ │ │ ├── BUILD.bazel │ │ │ ├── ClipClusterDeduplicationFilter.scala │ │ │ ├── ClusterBasedDedupFilter.scala │ │ │ ├── ConsistentAspectRatioFilter.scala │ │ │ ├── CountryFilter.scala │ │ │ ├── CurrentPinnedTweetFilter.scala │ │ │ ├── DropMaxCandidatesFilter.scala │ │ │ ├── FeedbackFatigueFilter.scala │ │ │ ├── GrokGoreFilter.scala │ │ │ ├── GrokNsfwFilter.scala │ │ │ ├── GrokSpamFilter.scala │ │ │ ├── GrokViolentFilter.scala │ │ │ ├── HasAuthorFilter.scala │ │ │ ├── HasMultipleMediaFilter.scala │ │ │ ├── InvalidConversationModuleFilter.scala │ │ │ ├── InvalidSubscriptionTweetFilter.scala │ │ │ ├── LocationFilter.scala │ │ │ ├── MaxVideoDurationFilter.scala │ │ │ ├── MediaDeduplicationFilter.scala │ │ │ ├── MinVideoDurationFilter.scala │ │ │ ├── PredicateGatedFilter.scala │ │ │ ├── PreviouslySeenMediaIdsFilter.scala │ │ │ ├── PreviouslySeenTweetsFilter.scala │ │ │ ├── PreviouslyServedAncestorsFilter.scala │ │ │ ├── PreviouslyServedTweetPreviewsFilter.scala │ │ │ ├── PreviouslyServedTweetsFilter.scala │ │ │ ├── QuoteDeduplicationFilter.scala │ │ │ ├── RegionFilter.scala │ │ │ ├── RejectTweetFromViewerFilter.scala │ │ │ ├── ReplyFilter.scala │ │ │ ├── RetweetDeduplicationFilter.scala │ │ │ ├── RetweetFilter.scala │ │ │ ├── SlopFilter.scala │ │ │ ├── TweetHydrationFilter.scala │ │ │ └── WeeklyBookmarkFilter.scala │ │ ├── gate/ │ │ │ ├── AllowForYouRecommendationsGate.scala │ │ │ ├── BUILD.bazel │ │ │ ├── BookmarksTimeGate.scala │ │ │ ├── DismissFatigueGate.scala │ │ │ ├── ExcludeSoftUserGate.scala │ │ │ ├── ExcludeSyntheticUserGate.scala │ │ │ ├── PersistenceStoreDurationValidationGate.scala │ │ │ ├── RateLimitGate.scala │ │ │ ├── RateLimitNotGate.scala │ │ │ ├── RecentlyServedByServedTypeGate.scala │ │ │ ├── RequestContextGate.scala │ │ │ ├── RequestContextNotGate.scala │ │ │ ├── SupportedLanguagesGate.scala │ │ │ ├── TestUserProbabilisticGate.scala │ │ │ └── TimelinesPersistenceStoreLastInjectionGate.scala │ │ ├── query_transformer/ │ │ │ ├── BUILD.bazel │ │ │ └── EditedTweetsCandidatePipelineQueryTransformer.scala │ │ ├── scorer/ │ │ │ ├── BUILD.bazel │ │ │ ├── FeedbackFatigueScorer.scala │ │ │ ├── NaviModelScorer.scala │ │ │ ├── OONTweetScalingScorer.scala │ │ │ ├── PhoenixModelRerankingScorer.scala │ │ │ ├── PhoenixScorer.scala │ │ │ ├── PredictClientFactory.scala │ │ │ └── WeighedModelRerankingScorer.scala │ │ ├── selector/ │ │ │ ├── BUILD.bazel │ │ │ ├── DebunchCandidates.scala │ │ │ ├── RandomShuffleCandidates.scala │ │ │ ├── ScoreAveragingPositionSelector.scala │ │ │ ├── SortFixedPositionCandidates.scala │ │ │ ├── UpdateConversationModuleId.scala │ │ │ ├── UpdateHomeClientEventDetails.scala │ │ │ └── UpdateNewTweetsPillDecoration.scala │ │ └── side_effect/ │ │ ├── BUILD.bazel │ │ ├── BaseCacheCandidateFeaturesSideEffect.scala │ │ ├── ClientEventsBuilder.scala │ │ ├── CommonFeaturesPldrConverter.scala │ │ ├── HomeScribeClientEventSideEffect.scala │ │ ├── HomeScribeServedCandidatesSideEffect.scala │ │ ├── PublishClientSentImpressionsEventBusSideEffect.scala │ │ ├── PublishClientSentImpressionsManhattanSideEffect.scala │ │ ├── PublishImpressionBloomFilterSideEffect.scala │ │ ├── TruncateTimelinesPersistenceStoreSideEffect.scala │ │ ├── UpdateLastNonPollingTimeSideEffect.scala │ │ └── UpdateTimelinesPersistenceStoreSideEffect.scala │ ├── marshaller/ │ │ ├── request/ │ │ │ ├── BUILD.bazel │ │ │ ├── DeviceContextUnmarshaller.scala │ │ │ ├── HomeMixerDebugParamsUnmarshaller.scala │ │ │ ├── HomeMixerProductContextUnmarshaller.scala │ │ │ ├── HomeMixerProductUnmarshaller.scala │ │ │ └── HomeMixerRequestUnmarshaller.scala │ │ ├── timeline_logging/ │ │ │ ├── BUILD.bazel │ │ │ ├── PromotedTweetDetailsMarshaller.scala │ │ │ ├── TweetDetailsMarshaller.scala │ │ │ └── WhoToFollowDetailsMarshaller.scala │ │ └── timelines/ │ │ ├── BUILD.bazel │ │ ├── ChronologicalCursorMarshaller.scala │ │ ├── ChronologicalCursorUnmarshaller.scala │ │ ├── DeviceContextMarshaller.scala │ │ ├── RecommendedUsersCursorUnmarshaller.scala │ │ ├── TimelineServiceCursorMarshaller.scala │ │ └── TopicContextFunctionalityTypeUnmarshaller.scala │ ├── model/ │ │ ├── BUILD.bazel │ │ ├── ClearCacheIncludeInstruction.scala │ │ ├── ContentFeatures.scala │ │ ├── GapIncludeInstruction.scala │ │ ├── GrokTopics.scala │ │ ├── HomeAdsQuery.scala │ │ ├── HomeFeatures.scala │ │ ├── HomeLargeEmbeddingsFeatures.scala │ │ ├── NaviClientConfig.scala │ │ ├── NavigationIncludeInstruction.scala │ │ ├── PhoenixPredictedScoreFeature.scala │ │ ├── PredictedScoreFeature.scala │ │ ├── candidate_source/ │ │ │ ├── BUILD.bazel │ │ │ └── SourceSignal.scala │ │ ├── request/ │ │ │ ├── BUILD.bazel │ │ │ ├── DeviceContext.scala │ │ │ ├── HasListId.scala │ │ │ ├── HasSeenTweetIds.scala │ │ │ ├── HomeMixerDebugOptions.scala │ │ │ ├── HomeMixerProduct.scala │ │ │ ├── HomeMixerProductContext.scala │ │ │ └── HomeMixerRequest.scala │ │ └── signup/ │ │ ├── BUILD.bazel │ │ └── SignupSource.scala │ ├── module/ │ │ ├── AdvertiserBrandSafetySettingsStoreModule.scala │ │ ├── BUILD.bazel │ │ ├── BlenderClientModule.scala │ │ ├── ClientSentImpressionsPublisherModule.scala │ │ ├── ClusterDetailsModule.scala │ │ ├── ConversationServiceModule.scala │ │ ├── EarlybirdRealtimeCGModule.scala │ │ ├── EventsRecosClientModule.scala │ │ ├── FeedbackHistoryClientModule.scala │ │ ├── GizmoduckTimelinesCacheClientModule.scala │ │ ├── HomeAdsCandidateSourceModule.scala │ │ ├── HomeMixerFeaturesModule.scala │ │ ├── HomeMixerFlagsModule.scala │ │ ├── HomeMixerResourcesModule.scala │ │ ├── ImpressionBloomFilterModule.scala │ │ ├── InMemoryCacheModule.scala │ │ ├── InjectionHistoryClientModule.scala │ │ ├── LimiterModule.scala │ │ ├── ManhattanClientsModule.scala │ │ ├── ManhattanFeatureRepositoryModule.scala │ │ ├── ManhattanTweetImpressionStoreModule.scala │ │ ├── MediaClusterId88Module.scala │ │ ├── MediaClusterIdModule.scala │ │ ├── MemcachedFeatureRepositoryModule.scala │ │ ├── MemcachedScoredCandidateFeaturesStoreModule.scala │ │ ├── NaviModelClientModule.scala │ │ ├── OptimizedStratoClientModule.scala │ │ ├── PeopleDiscoveryServiceModule.scala │ │ ├── PhoenixClientModule.scala │ │ ├── PipelineFailureExceptionMapper.scala │ │ ├── RealGraphInNetworkScoresModule.scala │ │ ├── RealtimeAggregateFeatureRepositoryModule.scala │ │ ├── ScoredTweetsMemcacheModule.scala │ │ ├── ScoredVideoTweetsMemcacheModule.scala │ │ ├── ScribeEventPublisherModule.scala │ │ ├── SimClustersRecentEngagementsClientModule.scala │ │ ├── StaleTweetsCacheModule.scala │ │ ├── ThriftFeatureRepositoryModule.scala │ │ ├── TimelinesPersistenceStoreClientModule.scala │ │ ├── TopicSocialProofClientModule.scala │ │ ├── TvWatchHistoryCacheClientModule.scala │ │ ├── TweetWatchTimeMetadataModule.scala │ │ ├── TweetyPieClientModule.scala │ │ ├── TweetypieStaticEntitiesCacheClientModule.scala │ │ ├── TwhinEmbeddingsModule.scala │ │ ├── UttTopicModule.scala │ │ └── VideoEmbeddingModule.scala │ ├── param/ │ │ ├── BUILD.bazel │ │ ├── GlobalParamConfigModule.scala │ │ ├── HomeGlobalParamConfig.scala │ │ ├── HomeGlobalParams.scala │ │ ├── HomeMixerFlagName.scala │ │ ├── HomeMixerInjectionNames.scala │ │ └── decider/ │ │ ├── BUILD.bazel │ │ └── DeciderKey.scala │ ├── product/ │ │ ├── BUILD.bazel │ │ ├── HomeMixerProductModule.scala │ │ ├── HomeProductPipelineRegistryConfig.scala │ │ ├── following/ │ │ │ ├── BUILD.bazel │ │ │ ├── FollowingAdsCandidatePipelineBuilder.scala │ │ │ ├── FollowingAdsDependentCandidatePipelineBuilder.scala │ │ │ ├── FollowingDependentAdsMixerPipelineConfig.scala │ │ │ ├── FollowingEarlybirdCandidatePipelineConfig.scala │ │ │ ├── FollowingEarlybirdQueryTransformer.scala │ │ │ ├── FollowingEarlybirdResponseFeatureTransformer.scala │ │ │ ├── FollowingMixerPipelineConfig.scala │ │ │ ├── FollowingProductPipelineConfig.scala │ │ │ ├── FollowingWhoToFollowCandidatePipelineConfigBuilder.scala │ │ │ ├── model/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── FollowingQuery.scala │ │ │ │ └── HomeMixerExternalStrings.scala │ │ │ └── param/ │ │ │ ├── BUILD.bazel │ │ │ ├── FollowingParam.scala │ │ │ └── FollowingParamConfig.scala │ │ ├── for_you/ │ │ │ ├── BUILD.bazel │ │ │ ├── ForYouAdsCandidatePipelineBuilder.scala │ │ │ ├── ForYouAdsDependentCandidatePipelineBuilder.scala │ │ │ ├── ForYouBookmarksCandidatePipelineConfig.scala │ │ │ ├── ForYouCommunitiesToJoinCandidatePipelineConfig.scala │ │ │ ├── ForYouConversationServiceCandidatePipelineConfig.scala │ │ │ ├── ForYouEntryPointPivotCandidatePipelineBuilder.scala │ │ │ ├── ForYouEntryPointPivotCandidatePipelineConfig.scala │ │ │ ├── ForYouExplorationTweetsCandidatePipelineConfig.scala │ │ │ ├── ForYouJetfuelFrameFrameCandidatePipelineConfig.scala │ │ │ ├── ForYouKeywordTrendsCandidatePipelineConfig.scala │ │ │ ├── ForYouMixerPipelineConfig.scala │ │ │ ├── ForYouPinnedTweetsCandidatePipelineConfig.scala │ │ │ ├── ForYouProductPipelineConfig.scala │ │ │ ├── ForYouPushToHomeMixerPipelineConfig.scala │ │ │ ├── ForYouPushToHomeTweetCandidatePipelineConfig.scala │ │ │ ├── ForYouRecommendedJobsCandidatePipelineConfig.scala │ │ │ ├── ForYouRecommendedRecruitingOrganizationsCandidatePipelineConfig.scala │ │ │ ├── ForYouRelevancePromptCandidatePipelineConfig.scala │ │ │ ├── ForYouResponseDomainMarshaller.scala │ │ │ ├── ForYouScoredTweetsCandidatePipelineConfig.scala │ │ │ ├── ForYouScoredTweetsMixerPipelineConfig.scala │ │ │ ├── ForYouScoredTweetsResponseFeatureTransformer.scala │ │ │ ├── ForYouScoredVideoTweetsCandidatePipelineConfig.scala │ │ │ ├── ForYouStoriesCandidatePipelineConfig.scala │ │ │ ├── ForYouTimelineScorerCandidatePipelineConfig.scala │ │ │ ├── ForYouTimelineScorerMixerPipelineConfig.scala │ │ │ ├── ForYouTimelineScorerResponseFeatureTransformer.scala │ │ │ ├── ForYouTuneFeedCandidatePipelineConfig.scala │ │ │ ├── ForYouTweetPreviewsCandidatePipelineConfig.scala │ │ │ ├── ForYouWhoToFollowCandidatePipelineConfigBuilder.scala │ │ │ ├── ForYouWhoToSubscribeCandidatePipelineConfigBuilder.scala │ │ │ ├── candidate_source/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── BookmarksCandidateSource.scala │ │ │ │ ├── BroadcastedPinnedTweetsCandidateSource.scala │ │ │ │ ├── JetfuelFrameCandidateSource.scala │ │ │ │ ├── RecommendedJobsCandidateSource.scala │ │ │ │ ├── RecommendedRecruitingOrganizationsCandidateSource.scala │ │ │ │ ├── ScoredTweetsProductCandidateSource.scala │ │ │ │ ├── ScoredVideoTweetsCategorizedProductCandidateSource.scala │ │ │ │ ├── ScoredVideoTweetsProductCandidateSource.scala │ │ │ │ ├── StoriesModuleCandidateSource.scala │ │ │ │ ├── TuneFeedCandidateSource.scala │ │ │ │ └── UnifiedTrendsCandidateSource.scala │ │ │ ├── feature_hydrator/ │ │ │ │ ├── ArticlePreviewTextFeatureHydrator.scala │ │ │ │ ├── AuthorEnabledPreviewsFeatureHydrator.scala │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── CurrentPinnedTweetFeatureHydrator.scala │ │ │ │ ├── DisplayedGrokTopicQueryFeatureHydrator.scala │ │ │ │ ├── FocalTweetFeatureHydrator.scala │ │ │ │ ├── FollowingSportsAccountQueryFeatureHydrator.scala │ │ │ │ ├── TimelineServiceTweetsQueryFeatureHydrator.scala │ │ │ │ ├── TweetAuthorFeatureHydrator.scala │ │ │ │ ├── TweetAuthorFollowersFeatureHydrator.scala │ │ │ │ ├── TweetEngagementsFeatureHydrator.scala │ │ │ │ ├── TweetPreviewTweetypieCandidateFeatureHydrator.scala │ │ │ │ └── ViewerHasJobRecommendationsFeatureHydrator.scala │ │ │ ├── filter/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── NotArticleFilter.scala │ │ │ │ ├── PromotedTrendFilter.scala │ │ │ │ ├── SocialContextFilter.scala │ │ │ │ └── TweetPreviewTextFilter.scala │ │ │ ├── functional_component/ │ │ │ │ └── gate/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── PushToHomeRequestGate.scala │ │ │ ├── gate/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── FollowingSportsUsersGate.scala │ │ │ │ ├── TuneFeedModuleGate.scala │ │ │ │ └── UserFollowingRangeGate.scala │ │ │ ├── model/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── ForYouQuery.scala │ │ │ │ └── ForYouTweetsResponse.scala │ │ │ ├── param/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── ForYouParam.scala │ │ │ │ └── ForYouParamConfig.scala │ │ │ ├── query_transformer/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── TweetPreviewsQueryTransformer.scala │ │ │ │ └── UnifiedCandidatesQueryTransformer.scala │ │ │ ├── response_transformer/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── BookmarksResponseFeatureTransformer.scala │ │ │ │ ├── ExplorationTweetResponseFeatureTransformer.scala │ │ │ │ ├── KeywordTrendsFeatureTransformer.scala │ │ │ │ ├── PinnedTweetResponseFeatureTransformer.scala │ │ │ │ ├── ScoredVideoTweetResponseFeatureTransformer.scala │ │ │ │ ├── StoriesModuleResponseFeatureTransformer.scala │ │ │ │ ├── TuneFeedFeatureTransformer.scala │ │ │ │ └── TweetPreviewResponseFeatureTransformer.scala │ │ │ ├── scorer/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── PinnedTweetCandidateScorer.scala │ │ │ ├── selector/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── DebugUpdateSortAdsResult.scala │ │ │ │ ├── DebunchCandidates.scala │ │ │ │ └── RemoveDuplicateCandidatesOutsideModule.scala │ │ │ └── side_effect/ │ │ │ ├── BUILD.bazel │ │ │ ├── ServedCandidateFeatureKeysKafkaSideEffect.scala │ │ │ ├── ServedCandidateFeatureKeysKafkaSideEffectBuilder.scala │ │ │ ├── ServedCandidateKafkaSideEffect.scala │ │ │ ├── ServedCandidateKeysKafkaSideEffect.scala │ │ │ ├── ServedCandidateKeysKafkaSideEffectBuilder.scala │ │ │ ├── ServedStatsSideEffect.scala │ │ │ └── VideoServedStatsSideEffect.scala │ │ ├── list_recommended_users/ │ │ │ ├── BUILD.bazel │ │ │ ├── BlenderUsersCandidatePipelineConfig.scala │ │ │ ├── BlenderUsersCandidatePipelineQueryTransformer.scala │ │ │ ├── ListMemberBasedUsersCandidatePipelineConfig.scala │ │ │ ├── ListMemberBasedUsersResponseFeatureTransfromer.scala │ │ │ ├── ListRecommendedUsersMixerPipelineConfig.scala │ │ │ ├── ListRecommendedUsersProductPipelineConfig.scala │ │ │ ├── candidate_source/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── BlenderUsersCandidateSource.scala │ │ │ │ └── SimilarityBasedUsersCandidateSource.scala │ │ │ ├── feature_hydrator/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── IsGizmoduckValidUserFeatureHydrator.scala │ │ │ │ ├── IsListMemberFeatureHydrator.scala │ │ │ │ ├── IsSGSValidUserFeatureHydrator.scala │ │ │ │ └── RecentListMembersQueryFeatureHydrator.scala │ │ │ ├── filter/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── DropMaxCandidatesByAggregatedScoreFilter.scala │ │ │ │ └── PreviouslyServedUsersFilter.scala │ │ │ ├── gate/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── ViewerIsListOwnerGate.scala │ │ │ ├── model/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── ListRecommendedUsersFeatures.scala │ │ │ │ └── ListRecommendedUsersQuery.scala │ │ │ └── param/ │ │ │ ├── BUILD.bazel │ │ │ ├── ListRecommendedUsersParam.scala │ │ │ └── ListRecommendedUsersParamConfig.scala │ │ ├── list_tweets/ │ │ │ ├── BUILD.bazel │ │ │ ├── ListTweetsAdsCandidatePipelineBuilder.scala │ │ │ ├── ListTweetsMixerPipelineConfig.scala │ │ │ ├── ListTweetsProductPipelineConfig.scala │ │ │ ├── ListTweetsTimelineServiceCandidatePipelineConfig.scala │ │ │ ├── decorator/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── ListConversationServiceCandidateDecorator.scala │ │ │ │ └── builder/ │ │ │ │ └── BUILD.bazel │ │ │ ├── model/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── ListTweetsQuery.scala │ │ │ └── param/ │ │ │ ├── BUILD.bazel │ │ │ ├── ListTweetsParam.scala │ │ │ └── ListTweetsParamConfig.scala │ │ ├── scored_tweets/ │ │ │ ├── BUILD.bazel │ │ │ ├── ScoredTweetsProductPipelineConfig.scala │ │ │ ├── ScoredTweetsRecommendationPipelineConfig.scala │ │ │ ├── candidate_pipeline/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── CachedScoredTweetsCandidatePipelineConfig.scala │ │ │ │ ├── ScoredTweetsBackfillCandidatePipelineConfig.scala │ │ │ │ ├── ScoredTweetsContentExplorationCandidatePipelineConfig.scala │ │ │ │ ├── ScoredTweetsDirectUtegCandidatePipelineConfig.scala │ │ │ │ ├── ScoredTweetsFrsCandidatePipelineConfig.scala │ │ │ │ ├── ScoredTweetsInNetworkCandidatePipelineConfig.scala │ │ │ │ ├── ScoredTweetsListsCandidatePipelineConfig.scala │ │ │ │ ├── ScoredTweetsPopularVideosCandidatePipelineConfig.scala │ │ │ │ ├── ScoredTweetsStaticCandidatePipelineConfig.scala │ │ │ │ ├── ScoredTweetsTweetMixerCandidatePipelineConfig.scala │ │ │ │ ├── ScoredTweetsUtegCandidatePipelineConfig.scala │ │ │ │ └── earlybird/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── ScoredTweetsCommunitiesCandidatePipelineConfig.scala │ │ │ │ ├── ScoredTweetsEarlybirdFrsCandidatePipelineConfig.scala │ │ │ │ └── ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.scala │ │ │ ├── candidate_source/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── CachedScoredTweetsCandidateSource.scala │ │ │ │ ├── ContentExplorationCandidateSource.scala │ │ │ │ ├── EarlybirdRealtimeCGTweetCandidateSource.scala │ │ │ │ ├── ListsCandidateSource.scala │ │ │ │ └── StaticPostsCandidateSource.scala │ │ │ ├── feature_hydrator/ │ │ │ │ ├── AncestorFeatureHydrator.scala │ │ │ │ ├── AuthorFeatureHydrator.scala │ │ │ │ ├── AuthorIsCreatorFeatureHydrator.scala │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── CachedScoredTweetsQueryFeatureHydrator.scala │ │ │ │ ├── EarlybirdFeatureHydrator.scala │ │ │ │ ├── FollowedUserScoresFeatureHydrator.scala │ │ │ │ ├── FrsSeedUsersQueryFeatureHydrator.scala │ │ │ │ ├── GizmoduckAuthorFeatureHydrator.scala │ │ │ │ ├── GraphTwoHopFeatureHydrator.scala │ │ │ │ ├── InvalidateCachedScoredTweetsQueryFeatureHydrator.scala │ │ │ │ ├── IsColdStartPostFeatureHydrator.scala │ │ │ │ ├── IsExtendedReplyFeatureHydrator.scala │ │ │ │ ├── ListIdsQueryFeatureHydrator.scala │ │ │ │ ├── ListNameFeatureHydrator.scala │ │ │ │ ├── LowSignalUserQueryFeatureHydrator.scala │ │ │ │ ├── MetricCenterUserCountingFeatureHydrator.scala │ │ │ │ ├── RealGraphQueryFeatureHydrator.scala │ │ │ │ ├── RealGraphViewerAuthorFeatureHydrator.scala │ │ │ │ ├── RealGraphViewerRelatedUsersFeatureHydrator.scala │ │ │ │ ├── RealTimeInteractionGraphEdgeFeatureHydrator.scala │ │ │ │ ├── RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala │ │ │ │ ├── ReplyFeatureHydrator.scala │ │ │ │ ├── RequestTimeQueryFeatureHydrator.scala │ │ │ │ ├── RetweetSourceTweetFeatureHydrator.scala │ │ │ │ ├── SGSMutuallyFollowedUserHydrator.scala │ │ │ │ ├── SemanticCoreFeatureHydrator.scala │ │ │ │ ├── SimClustersEngagementSimilarityFeatureHydrator.scala │ │ │ │ ├── SimClustersUserTweetScoresHydrator.scala │ │ │ │ ├── TSPInferredTopicFeatureHydrator.scala │ │ │ │ ├── TweetMetaDataFeatureHydrator.scala │ │ │ │ ├── TweetTimeFeatureHydrator.scala │ │ │ │ ├── TweetypieContentFeatureHydrator.scala │ │ │ │ ├── TweetypieStaticEntitiesFeatureHydrator.scala │ │ │ │ ├── TweetypieVisibilityFeatureHydrator.scala │ │ │ │ ├── TwhinAuthorFollowFeatureHydrator.scala │ │ │ │ ├── TwhinUserEngagementQueryFeatureHydrator.scala │ │ │ │ ├── TwhinUserFollowQueryFeatureHydrator.scala │ │ │ │ ├── UserFollowedTopicIdsFeatureHydrator.scala │ │ │ │ ├── UserLanguagesFeatureHydrator.scala │ │ │ │ ├── UserStateQueryFeatureHydrator.scala │ │ │ │ ├── UtegFeatureHydrator.scala │ │ │ │ ├── ValidLikedByUserIdsFeatureHydrator.scala │ │ │ │ ├── adapters/ │ │ │ │ │ ├── author_features/ │ │ │ │ │ │ ├── AuthorFeaturesAdapter.scala │ │ │ │ │ │ └── BUILD.bazel │ │ │ │ │ ├── content/ │ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ │ ├── ContentFeatureAdapter.scala │ │ │ │ │ │ └── InReplyToContentFeatureAdapter.scala │ │ │ │ │ ├── earlybird/ │ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ │ ├── EarlybirdAdapter.scala │ │ │ │ │ │ └── InReplyToEarlybirdAdapter.scala │ │ │ │ │ ├── inferred_topic/ │ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ │ └── InferredTopicAdapter.scala │ │ │ │ │ ├── non_ml_features/ │ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ │ ├── NonMLCandidateFeaturesAdapter.scala │ │ │ │ │ │ └── NonMLCommonFeaturesAdapter.scala │ │ │ │ │ ├── offline_aggregates/ │ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ │ ├── PassThroughAdapter.scala │ │ │ │ │ │ └── SparseAggregatesToDenseAdapter.scala │ │ │ │ │ └── twhin_embeddings/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ └── TwhinEmbeddingsAdapter.scala │ │ │ │ ├── offline_aggregates/ │ │ │ │ │ ├── AggregateFeatureInfo.scala │ │ │ │ │ ├── AggregateFeaturesToDecodeWithMetadata.scala │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ ├── BaseAggregateQueryFeatureHydrator.scala │ │ │ │ │ ├── BaseEdgeAggregateFeatureHydrator.scala │ │ │ │ │ ├── EdgeAggregateFeatures.scala │ │ │ │ │ ├── PartAAggregateQueryFeatureHydrator.scala │ │ │ │ │ ├── PartBAggregateQueryFeatureHydrator.scala │ │ │ │ │ ├── Phase1EdgeAggregateFeatureHydrator.scala │ │ │ │ │ ├── Phase2EdgeAggregateFeatureHydrator.scala │ │ │ │ │ └── Utils.scala │ │ │ │ └── real_time_aggregates/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── BaseRealTimeAggregateBulkCandidateFeatureHydrator.scala │ │ │ │ ├── BaseRealTimeAggregateQueryFeatureHydrator.scala │ │ │ │ ├── BaseRealtimeAggregateHydrator.scala │ │ │ │ ├── EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala │ │ │ │ ├── RealTimeAggregateTimeDecay.scala │ │ │ │ ├── TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala │ │ │ │ ├── TopicEngagementRealTimeAggregateFeatureHydrator.scala │ │ │ │ ├── TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala │ │ │ │ ├── TweetEngagementRealTimeAggregateFeatureHydrator.scala │ │ │ │ ├── TwitterListEngagementRealTimeAggregateFeatureHydrator.scala │ │ │ │ ├── UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala │ │ │ │ └── UserEngagementRealTimeAggregatesFeatureHydrator.scala │ │ │ ├── filter/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── ControlAiExcludeFilter.scala │ │ │ │ ├── ControlAiOnlyIncludeFilter.scala │ │ │ │ ├── CustomSnowflakeIdAgeFilter.scala │ │ │ │ ├── CustomSnowflakeIdAgeFilterWithIncludeRule.scala │ │ │ │ ├── CustomSnowflakeIdAgeFilterWithSkipRule.scala │ │ │ │ ├── DuplicateConversationTweetsFilter.scala │ │ │ │ ├── ExtendedDirectedAtFilter.scala │ │ │ │ ├── FollowedAuthorFilter.scala │ │ │ │ ├── GrokAutoTranslateLanguageFilter.scala │ │ │ │ ├── IsOutOfNetworkColdStartPostFilter.scala │ │ │ │ ├── LanguageFilter.scala │ │ │ │ ├── OONReplyFilter.scala │ │ │ │ ├── OutOfNetworkCompetitorFilter.scala │ │ │ │ ├── OutOfNetworkCompetitorURLFilter.scala │ │ │ │ ├── QualifiedRepliesFilter.scala │ │ │ │ ├── RetweetSourceTweetRemovingFilter.scala │ │ │ │ ├── SGSAuthorFilter.scala │ │ │ │ ├── ScoredTweetsSocialContextFilter.scala │ │ │ │ ├── TopKFilter.scala │ │ │ │ ├── TopKOptionalFilter.scala │ │ │ │ ├── UtegMinFavCountFilter.scala │ │ │ │ └── UtegTopKFilter.scala │ │ │ ├── gate/ │ │ │ │ ├── AllowLowSignalUserGate.scala │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── DenyLowSignalUserGate.scala │ │ │ │ ├── MatchesCountryGate.scala │ │ │ │ ├── MinCachedTweetsGate.scala │ │ │ │ ├── MinTimeSinceLastRequestGate.scala │ │ │ │ └── RecentFeedbackCheckGate.scala │ │ │ ├── marshaller/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── ScoredTweetsResponseDomainMarshaller.scala │ │ │ │ └── ScoredTweetsResponseTransportMarshaller.scala │ │ │ ├── model/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── ScoredTweetsQuery.scala │ │ │ │ └── ScoredTweetsResponse.scala │ │ │ ├── param/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── ScoredTweetsParam.scala │ │ │ │ └── ScoredTweetsParamConfig.scala │ │ │ ├── query_transformer/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── ContentExplorationQueryTransformer.scala │ │ │ │ ├── TimelineRankerFrsQueryTransformer.scala │ │ │ │ ├── TimelineRankerInNetworkQueryTransformer.scala │ │ │ │ ├── TimelineRankerQueryTransformer.scala │ │ │ │ ├── TimelineRankerUtegQueryTransformer.scala │ │ │ │ ├── UtegQueryTransformer.scala │ │ │ │ └── earlybird/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── CommunitiesEarlybirdQueryTransformer.scala │ │ │ │ ├── EarlybirdFrsQueryTransformer.scala │ │ │ │ ├── EarlybirdInNetworkQueryTransformer.scala │ │ │ │ └── EarlybirdQueryTransformer.scala │ │ │ ├── response_transformer/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── CachedScoredTweetsResponseFeatureTransformer.scala │ │ │ │ ├── ScoredTweetsBackfillResponseFeatureTransformer.scala │ │ │ │ ├── ScoredTweetsContentExplorationResponseFeatureTransformer.scala │ │ │ │ ├── ScoredTweetsDirectUtegResponseFeatureTransformer.scala │ │ │ │ ├── ScoredTweetsFrsResponseFeatureTransformer.scala │ │ │ │ ├── ScoredTweetsInNetworkResponseFeatureTransformer.scala │ │ │ │ ├── ScoredTweetsListsResponseFeatureTransformer.scala │ │ │ │ ├── ScoredTweetsOfflineVideoRecoResponseFeatureTransformer.scala │ │ │ │ ├── ScoredTweetsPopularVideosResponseFeatureTransformer.scala │ │ │ │ ├── ScoredTweetsStaticResponseFeatureTransformer.scala │ │ │ │ ├── ScoredTweetsTweetMixerResponseFeatureTransformer.scala │ │ │ │ ├── ScoredTweetsUtegResponseFeatureTransformer.scala │ │ │ │ ├── ScoredVideoTweetsPinnedTweetResponseFeatureTransformer.scala │ │ │ │ ├── TimelineRankerResponseTransformer.scala │ │ │ │ └── earlybird/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── EarlybirdResponseTransformer.scala │ │ │ │ ├── ScoredTweetsCommunitiesResponseFeatureTransformer.scala │ │ │ │ ├── ScoredTweetsEarlybirdFrsResponseFeatureTransformer.scala │ │ │ │ ├── ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer.scala │ │ │ │ └── ScoredTweetsEarlybirdNsfwResponseFeatureTransformer.scala │ │ │ ├── scorer/ │ │ │ │ ├── AuthorBasedListwiseRescoringProvider.scala │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── CandidateSourceDiversityListwiseRescoringProvider.scala │ │ │ │ ├── ContentExplorationListwiseRescoringProvider.scala │ │ │ │ ├── ControlAiRescorer.scala │ │ │ │ ├── DeepRetrievalListwiseRescoringProvider.scala │ │ │ │ ├── DiversityDiscountProvider.scala │ │ │ │ ├── EvergreenDeepRetrievalCrossBorderListwiseRescoringProvider.scala │ │ │ │ ├── EvergreenDeepRetrievalListwiseRescoringProvider.scala │ │ │ │ ├── GrokSlopScoreRescorer.scala │ │ │ │ ├── HeuristicScorer.scala │ │ │ │ ├── ImpressedAuthorDecayRescoringProvider.scala │ │ │ │ ├── ImpressedImageClusterBasedListwiseRescoringProvider.scala │ │ │ │ ├── ImpressedMediaClusterBasedListwiseRescoringProvider.scala │ │ │ │ ├── ListwiseRescoringProvider.scala │ │ │ │ ├── LowSignalScorer.scala │ │ │ │ ├── MultimodalEmbeddingRescorer.scala │ │ │ │ ├── NaviModelScorer.scala │ │ │ │ ├── PredictedScoreFeature.scala │ │ │ │ └── RescoringFactorProvider.scala │ │ │ ├── scoring_pipeline/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── ScoredTweetsHeuristicScoringPipelineConfig.scala │ │ │ │ ├── ScoredTweetsLowSignalScoringPipelineConfig.scala │ │ │ │ ├── ScoredTweetsModelScoringPipelineConfig.scala │ │ │ │ └── ScoredTweetsRerankingScoringPipelineConfig.scala │ │ │ ├── selector/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType.scala │ │ │ │ └── KeepTopKCandidatesPerCommunity.scala │ │ │ ├── side_effect/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── CacheCandidateFeaturesSideEffect.scala │ │ │ │ ├── CacheRequestInfoSideEffect.scala │ │ │ │ ├── CacheRetrievalSignalSideEffect.scala │ │ │ │ ├── CachedScoredTweetsSideEffect.scala │ │ │ │ ├── CommonFeaturesSideEffect.scala │ │ │ │ ├── ScoredCandidateFeatureKeysKafkaSideEffect.scala │ │ │ │ ├── ScoredContentExplorationCandidateScoreFeatureKafkaSideEffect.scala │ │ │ │ ├── ScoredPhoenixCandidatesKafkaSideEffect.scala │ │ │ │ ├── ScoredStatsSideEffect.scala │ │ │ │ ├── ScoredTweetsDiversityStatsSideEffect.scala │ │ │ │ ├── ScribeScoredCandidatesSideEffect.scala │ │ │ │ └── ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect.scala │ │ │ └── util/ │ │ │ ├── BUILD.bazel │ │ │ └── ControlAiUtil.scala │ │ └── subscribed/ │ │ ├── BUILD.bazel │ │ ├── SubscribedEarlybirdCandidatePipelineConfig.scala │ │ ├── SubscribedEarlybirdQueryTransformer.scala │ │ ├── SubscribedEarlybirdResponseFeatureTransformer.scala │ │ ├── SubscribedMixerPipelineConfig.scala │ │ ├── SubscribedProductPipelineConfig.scala │ │ ├── model/ │ │ │ ├── BUILD.bazel │ │ │ └── SubscribedQuery.scala │ │ └── param/ │ │ ├── BUILD.bazel │ │ ├── SubscribedParam.scala │ │ └── SubscribedParamConfig.scala │ ├── service/ │ │ ├── BUILD.bazel │ │ ├── HomeMixerAccessPolicy.scala │ │ ├── HomeMixerAlertConfig.scala │ │ └── ScoredTweetsService.scala │ ├── store/ │ │ ├── BUILD.bazel │ │ ├── MediaClusterId88Store.scala │ │ ├── MediaClusterId95Store.scala │ │ ├── MediaClusterIdStoreTrait.scala │ │ ├── RTAMHStore.scala │ │ ├── RealGraphInNetworkScoresStore.scala │ │ ├── TweetWatchTimeMetadataStore.scala │ │ ├── TwhinEmbeddingsStore.scala │ │ └── VideoEmbeddingMHStore.scala │ └── util/ │ ├── BUILD.bazel │ ├── CachedScoredTweetsHelper.scala │ ├── CandidatesUtil.scala │ ├── DataRecordUtil.scala │ ├── InjectionTransformer.scala │ ├── LanguageCode.scala │ ├── LanguageUtil.scala │ ├── MissingKeyException.scala │ ├── NaviScorerStatsHandler.scala │ ├── ObservedKeyValueResultHandler.scala │ ├── PhoenixScorerStatsHandler.scala │ ├── PhoenixUtils.scala │ ├── ReplyRetweetUtil.scala │ ├── RerankerUtil.scala │ ├── SignalUtil.scala │ ├── TensorFlowUtil.scala │ ├── TweetImpressionsHelper.scala │ ├── UrtUtil.scala │ ├── earlybird/ │ │ ├── BUILD.bazel │ │ ├── EarlybirdRequestUtil.scala │ │ ├── EarlybirdResponseUtil.scala │ │ └── RelevanceSearchUtil.scala │ └── tweetypie/ │ ├── BUILD.bazel │ ├── RequestFields.scala │ └── content/ │ ├── BUILD.bazel │ ├── FeatureExtractionHelper.scala │ ├── TweetMediaFeaturesExtractor.scala │ └── TweetTextFeaturesExtractor.scala ├── navi/ │ ├── README.md │ ├── dr_transform/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── all_config.rs │ │ ├── converter.rs │ │ ├── lib.rs │ │ └── util.rs │ ├── navi/ │ │ ├── Cargo.toml │ │ ├── build.rs │ │ ├── proto/ │ │ │ ├── kfserving/ │ │ │ │ └── grpc_predict_v2.proto │ │ │ ├── tensorflow/ │ │ │ │ └── core/ │ │ │ │ ├── example/ │ │ │ │ │ ├── example.proto │ │ │ │ │ └── feature.proto │ │ │ │ ├── framework/ │ │ │ │ │ ├── allocation_description.proto │ │ │ │ │ ├── api_def.proto │ │ │ │ │ ├── attr_value.proto │ │ │ │ │ ├── cost_graph.proto │ │ │ │ │ ├── dataset_metadata.proto │ │ │ │ │ ├── dataset_options.proto │ │ │ │ │ ├── device_attributes.proto │ │ │ │ │ ├── full_type.proto │ │ │ │ │ ├── function.proto │ │ │ │ │ ├── graph.proto │ │ │ │ │ ├── graph_transfer_info.proto │ │ │ │ │ ├── kernel_def.proto │ │ │ │ │ ├── log_memory.proto │ │ │ │ │ ├── model.proto │ │ │ │ │ ├── node_def.proto │ │ │ │ │ ├── op_def.proto │ │ │ │ │ ├── reader_base.proto │ │ │ │ │ ├── resource_handle.proto │ │ │ │ │ ├── step_stats.proto │ │ │ │ │ ├── summary.proto │ │ │ │ │ ├── tensor.proto │ │ │ │ │ ├── tensor_description.proto │ │ │ │ │ ├── tensor_shape.proto │ │ │ │ │ ├── tensor_slice.proto │ │ │ │ │ ├── types.proto │ │ │ │ │ ├── variable.proto │ │ │ │ │ └── versions.proto │ │ │ │ └── protobuf/ │ │ │ │ ├── autotuning.proto │ │ │ │ ├── bfc_memory_map.proto │ │ │ │ ├── cluster.proto │ │ │ │ ├── composite_tensor_variant.proto │ │ │ │ ├── config.proto │ │ │ │ ├── control_flow.proto │ │ │ │ ├── conv_autotuning.proto │ │ │ │ ├── coordination_config.proto │ │ │ │ ├── coordination_service.proto │ │ │ │ ├── critical_section.proto │ │ │ │ ├── data_service.proto │ │ │ │ ├── debug.proto │ │ │ │ ├── debug_event.proto │ │ │ │ ├── device_filters.proto │ │ │ │ ├── device_properties.proto │ │ │ │ ├── distributed_runtime_payloads.proto │ │ │ │ ├── eager_service.proto │ │ │ │ ├── error_codes.proto │ │ │ │ ├── graph_debug_info.proto │ │ │ │ ├── master.proto │ │ │ │ ├── master_service.proto │ │ │ │ ├── meta_graph.proto │ │ │ │ ├── named_tensor.proto │ │ │ │ ├── queue_runner.proto │ │ │ │ ├── remote_tensor_handle.proto │ │ │ │ ├── replay_log.proto │ │ │ │ ├── rewriter_config.proto │ │ │ │ ├── saved_model.proto │ │ │ │ ├── saved_object_graph.proto │ │ │ │ ├── saver.proto │ │ │ │ ├── service_config.proto │ │ │ │ ├── snapshot.proto │ │ │ │ ├── status.proto │ │ │ │ ├── struct.proto │ │ │ │ ├── tensor_bundle.proto │ │ │ │ ├── tensorflow_server.proto │ │ │ │ ├── trackable_object_graph.proto │ │ │ │ ├── transport_options.proto │ │ │ │ ├── verifier_config.proto │ │ │ │ ├── worker.proto │ │ │ │ └── worker_service.proto │ │ │ └── tensorflow_serving/ │ │ │ ├── apis/ │ │ │ │ ├── classification.proto │ │ │ │ ├── get_model_metadata.proto │ │ │ │ ├── get_model_status.proto │ │ │ │ ├── inference.proto │ │ │ │ ├── input.proto │ │ │ │ ├── logging.proto │ │ │ │ ├── model.proto │ │ │ │ ├── model_management.proto │ │ │ │ ├── model_service.proto │ │ │ │ ├── predict.proto │ │ │ │ ├── prediction_log.proto │ │ │ │ ├── prediction_service.proto │ │ │ │ ├── regression.proto │ │ │ │ ├── session_service.proto │ │ │ │ └── status.proto │ │ │ └── config/ │ │ │ ├── file_system_storage_path_source.proto │ │ │ ├── log_collector_config.proto │ │ │ ├── logging_config.proto │ │ │ └── model_server_config.proto │ │ ├── scripts/ │ │ │ ├── run_onnx.sh │ │ │ └── run_tf2.sh │ │ └── src/ │ │ ├── batch.rs │ │ ├── bin/ │ │ │ ├── navi.rs │ │ │ ├── navi_onnx.rs │ │ │ └── navi_torch.rs │ │ ├── bootstrap.rs │ │ ├── cli_args.rs │ │ ├── cores/ │ │ │ └── validator.rs │ │ ├── lib.rs │ │ ├── metrics.rs │ │ ├── onnx_model.rs │ │ ├── predict_service.rs │ │ ├── tf_model.rs │ │ └── torch_model.rs │ ├── segdense/ │ │ ├── Cargo.toml │ │ └── src/ │ │ ├── error.rs │ │ ├── lib.rs │ │ ├── main.rs │ │ ├── mapper.rs │ │ ├── segdense_transform_spec_home_recap_2022.rs │ │ └── util.rs │ └── thrift_bpr_adapter/ │ └── thrift/ │ ├── Cargo.toml │ └── src/ │ ├── data.rs │ ├── decoder.rs │ ├── lib.rs │ ├── main.rs │ ├── prediction_service.rs │ └── tensor.rs ├── product-mixer/ │ ├── README.md │ ├── component-library/ │ │ └── src/ │ │ └── main/ │ │ └── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── product_mixer/ │ │ └── component_library/ │ │ ├── candidate_source/ │ │ │ ├── account_recommendations_mixer/ │ │ │ │ ├── AccountRecommendationsMixerCandidateSource.scala │ │ │ │ └── BUILD │ │ │ ├── ads/ │ │ │ │ ├── AdsProdStratoCandidateSource.scala │ │ │ │ ├── AdsProdThriftCandidateSource.scala │ │ │ │ ├── AdsStagingCandidateSource.scala │ │ │ │ └── BUILD │ │ │ ├── ann/ │ │ │ │ ├── AnnCandidateSource.scala │ │ │ │ ├── AnnIdQuery.scala │ │ │ │ └── BUILD.bazel │ │ │ ├── audiospace/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── CreatedSpacesCandidateSource.scala │ │ │ ├── business_profiles/ │ │ │ │ ├── BUILD │ │ │ │ └── TeamMembersCandidateSource.scala │ │ │ ├── cr_mixer/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── CrMixerFrsBasedTweetRecommendationsCandidateSource.scala │ │ │ │ └── CrMixerTweetRecommendationsCandidateSource.scala │ │ │ ├── earlybird/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── EarlybirdTweetCandidateSource.scala │ │ │ ├── explore_ranker/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── ExploreRankerCandidateSource.scala │ │ │ ├── flexible_injection_pipeline/ │ │ │ │ ├── BUILD │ │ │ │ └── PromptCandidateSource.scala │ │ │ ├── hermit/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── UsersSimilarToMeCandidateSource.scala │ │ │ ├── interest_discovery/ │ │ │ │ ├── BUILD │ │ │ │ └── RelatedTopicsCandidateSource.scala │ │ │ ├── lists/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── OrganicPopGeoListsCandidateSource.scala │ │ │ ├── people_discovery/ │ │ │ │ ├── BUILD │ │ │ │ └── PeopleDiscoveryCandidateSource.scala │ │ │ ├── recommendations/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── UserFollowRecommendationsCandidateSource.scala │ │ │ ├── social_graph/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── SocialgraphCandidateSource.scala │ │ │ │ └── SocialgraphCursorConstants.scala │ │ │ ├── timeline_ranker/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── TimelineRankerInNetworkCandidateSource.scala │ │ │ │ ├── TimelineRankerRecapCandidateSource.scala │ │ │ │ └── TimelineRankerUtegCandidateSource.scala │ │ │ ├── timeline_scorer/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── TimelineScorerCandidateSource.scala │ │ │ ├── timeline_service/ │ │ │ │ ├── BUILD │ │ │ │ └── TimelineServiceTweetCandidateSource.scala │ │ │ ├── timelines_impression_store/ │ │ │ │ ├── BUILD │ │ │ │ └── TimelinesImpressionStoreCandidateSourceV2.scala │ │ │ ├── topics/ │ │ │ │ ├── BUILD │ │ │ │ └── FollowedTopicsCandidateSource.scala │ │ │ └── tweetconvosvc/ │ │ │ ├── BUILD.bazel │ │ │ ├── ConversationServiceCandidateSource.scala │ │ │ ├── ConversationServiceResponseFeatureTransformer.scala │ │ │ └── DropMaxConversationModuleItemCandidates.scala │ │ ├── decorator/ │ │ │ ├── slice/ │ │ │ │ ├── BUILD │ │ │ │ ├── SliceItemCandidateDecorator.scala │ │ │ │ └── builder/ │ │ │ │ ├── BUILD │ │ │ │ └── CursorCandidateSliceItemBuilder.scala │ │ │ └── urt/ │ │ │ ├── BUILD │ │ │ ├── UrtConversationItemCandidateDecorator.scala │ │ │ ├── UrtItemCandidateDecorator.scala │ │ │ ├── UrtItemInModuleDecorator.scala │ │ │ ├── UrtMultipleModulesDecorator.scala │ │ │ └── builder/ │ │ │ ├── contextual_ref/ │ │ │ │ └── ContextualTweetRefBuilder.scala │ │ │ ├── conversations/ │ │ │ │ └── ConversationModuleMetadataBuilder.scala │ │ │ ├── flexible_injection_pipeline/ │ │ │ │ ├── FlipPromptCandidateUrtItemBuilder.scala │ │ │ │ ├── FlipPromptModuleGrouping.scala │ │ │ │ ├── FlipPromptUrtModuleBuilder.scala │ │ │ │ ├── OnboardingInjectionConversions.scala │ │ │ │ ├── RelevancePromptConversions.scala │ │ │ │ └── TilesCarouselConversions.scala │ │ │ ├── icon/ │ │ │ │ └── HorizonIconBuilder.scala │ │ │ ├── item/ │ │ │ │ ├── ad/ │ │ │ │ │ └── AdsCandidateUrtItemBuilder.scala │ │ │ │ ├── alert/ │ │ │ │ │ ├── DurationParamBuilder.scala │ │ │ │ │ ├── ShowAlertCandidateUrtItemBuilder.scala │ │ │ │ │ ├── StaticShowAlertColorConfigurationBuilder.scala │ │ │ │ │ ├── StaticShowAlertDisplayLocationBuilder.scala │ │ │ │ │ └── StaticShowAlertIconDisplayInfoBuilder.scala │ │ │ │ ├── article/ │ │ │ │ │ └── ArticleCandidateUrtItemBuilder.scala │ │ │ │ ├── audio_space/ │ │ │ │ │ └── AudioSpaceCandidateUrtItemBuilder.scala │ │ │ │ ├── card/ │ │ │ │ │ └── CardCandidateUtrItemBuilder.scala │ │ │ │ ├── commerce/ │ │ │ │ │ ├── CommerceProductCandidateUrtItemBuilder.scala │ │ │ │ │ └── CommerceProductGroupCandidateUrtItemBuilder.scala │ │ │ │ ├── event_summary/ │ │ │ │ │ └── EventCandidateUrtItemBuilder.scala │ │ │ │ ├── generic_summary/ │ │ │ │ │ ├── GenericSummaryActionBuilder.scala │ │ │ │ │ ├── GenericSummaryCandidateUrtItemBuilder.scala │ │ │ │ │ └── GenericSummaryContextBuilder.scala │ │ │ │ ├── icon_label/ │ │ │ │ │ └── IconLabelCandidateUrtItemBuilder.scala │ │ │ │ ├── message/ │ │ │ │ │ ├── CompactPromptCandidateUrtItemStringCenterBuilder.scala │ │ │ │ │ ├── InlinePromptCandidateUrtItemStringCenterBuilder.scala │ │ │ │ │ ├── MessageTextActionBuilder.scala │ │ │ │ │ └── UserFacePileBuilder.scala │ │ │ │ ├── moment/ │ │ │ │ │ └── MomentAnnotationCandidateUrtItemBuilder.scala │ │ │ │ ├── relevance_prompt/ │ │ │ │ │ └── RelevancePromptCandidateUrtItemStringCenterBuilder.scala │ │ │ │ ├── suggestion/ │ │ │ │ │ └── SpellingSuggestionCandidateUrtItemBuilder.scala │ │ │ │ ├── tile/ │ │ │ │ │ └── TileCandidateUrtItemBuilder.scala │ │ │ │ ├── topic/ │ │ │ │ │ ├── ParamTopicDisplayTypeBuilder.scala │ │ │ │ │ ├── ParamTopicFunctionalityTypeBuilder.scala │ │ │ │ │ ├── StaticTopicDisplayTypeBuilder.scala │ │ │ │ │ ├── StaticTopicFunctionalityTypeBuilder.scala │ │ │ │ │ ├── TopicCandidateUrtItemBuilder.scala │ │ │ │ │ └── VerticalGridTopicCandidateUrtItemBuilder.scala │ │ │ │ ├── trend/ │ │ │ │ │ ├── TrendCandidateUrtItemBuilder.scala │ │ │ │ │ ├── TrendMetaDescriptionBuilder.scala │ │ │ │ │ └── TrendPromotedMetadataBuilder.scala │ │ │ │ ├── tweet/ │ │ │ │ │ └── TweetCandidateUrtItemBuilder.scala │ │ │ │ ├── twitter_list/ │ │ │ │ │ └── TwitterListCandidateUrtItemBuilder.scala │ │ │ │ ├── unified_trend_event/ │ │ │ │ │ └── UnifiedTrendEventCandidateUrtItemBuilder.scala │ │ │ │ └── user/ │ │ │ │ └── UserCandidateUrtItemBuilder.scala │ │ │ ├── metadata/ │ │ │ │ ├── ClientEventInfoBuilder.scala │ │ │ │ ├── ConversationTweetClientEventDetailsBuilder.scala │ │ │ │ ├── StaticUrlBuilder.scala │ │ │ │ ├── TopicClientEventDetailsBuilder.scala │ │ │ │ ├── TopicNotInterestedFeedbackActionInfoBuilder.scala │ │ │ │ ├── TopicTweetClientEventDetailsBuilder.scala │ │ │ │ ├── TopicsToFollowModuleMetadataBuilder.scala │ │ │ │ └── WhoToFollowFeedbackActionInfoBuilder.scala │ │ │ ├── operation/ │ │ │ │ └── CursorCandidateUrtOperationBuilder.scala │ │ │ ├── promoted/ │ │ │ │ └── FeaturePromotedMetadataBuilder.scala │ │ │ ├── richtext/ │ │ │ │ ├── RichTextBuilder.scala │ │ │ │ ├── RichTextMarkupUtil.scala │ │ │ │ ├── RichTextReferenceObjectBuilder.scala │ │ │ │ ├── RichTextRtlOptionBuilder.scala │ │ │ │ ├── StaticRichTextBuilder.scala │ │ │ │ └── twitter_text/ │ │ │ │ ├── TwitterTextEntityProcessor.scala │ │ │ │ ├── TwitterTextFormatProcessor.scala │ │ │ │ ├── TwitterTextRenderer.scala │ │ │ │ ├── TwitterTextRendererProcessor.scala │ │ │ │ └── TwitterTextRichTextBuilder.scala │ │ │ ├── social_context/ │ │ │ │ ├── FeatureSocialContextBuilder.scala │ │ │ │ ├── GeneralModuleSocialContextBuilder.scala │ │ │ │ ├── GeneralSocialContextBuilder.scala │ │ │ │ └── WhoToFollowSocialContextBuilder.scala │ │ │ ├── stringcenter/ │ │ │ │ ├── ModuleStr.scala │ │ │ │ └── Str.scala │ │ │ └── timeline_module/ │ │ │ ├── FeatureModuleDisplayTypeBuilder.scala │ │ │ ├── ModuleDynamicShowMoreBehaviorRevealByCountBuilder.scala │ │ │ ├── ModuleFooterBuilder.scala │ │ │ ├── ModuleHeaderBuilder.scala │ │ │ ├── ModuleHeaderDisplayTypeBuilder.scala │ │ │ ├── ModuleIdGeneration.scala │ │ │ ├── ModuleShowMoreBehaviorRevealByCountBuilder.scala │ │ │ ├── ParamGatedModuleFooterBuilder.scala │ │ │ ├── ParamGatedModuleHeaderBuilder.scala │ │ │ ├── ParamWhoToFollowModuleDisplayTypeBuilder.scala │ │ │ ├── StaticModuleDisplayTypeBuilder.scala │ │ │ └── TimelineModuleBuilder.scala │ │ ├── experiments/ │ │ │ └── metrics/ │ │ │ ├── BUILD │ │ │ ├── MetricDefinitions.scala │ │ │ ├── MetricGroup.scala │ │ │ ├── MetricTemplateCLIRunner.scala │ │ │ ├── MetricTemplates.scala │ │ │ └── PlaceholderConfig.scala │ │ ├── feature/ │ │ │ └── featurestorev1/ │ │ │ ├── BUILD │ │ │ ├── FeatureStoreV1QueryUserIdFeature.scala │ │ │ ├── FeatureStoreV1QueryUserIdTweetCandidateAuthorIdFeature.scala │ │ │ ├── FeatureStoreV1QueryUserIdTweetCandidateTweetIdFeature.scala │ │ │ ├── FeatureStoreV1TweetCandidateAuthorIdFeature.scala │ │ │ ├── FeatureStoreV1TweetCandidateTweetIdFeature.scala │ │ │ └── FeatureStoreV1UserCandidateUserIdFeature.scala │ │ ├── feature_hydrator/ │ │ │ ├── BUILD │ │ │ ├── candidate/ │ │ │ │ ├── ads/ │ │ │ │ │ ├── AdvertiserBrandSafetySettingsFeatureHydrator.scala │ │ │ │ │ └── BUILD │ │ │ │ ├── decay/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ └── DecayCandidateFeatureHydrator.scala │ │ │ │ ├── param_gated/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── ParamGatedBulkCandidateFeatureHydrator.scala │ │ │ │ │ ├── ParamGatedCandidateFeatureHydrator.scala │ │ │ │ │ └── featurestorev1/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── ParamGatedFeatureStoreV1CandidateFeatureHydrator.scala │ │ │ │ ├── qualityfactor_gated/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ └── QualityFactorGatedCandidateFeatureHydrator.scala │ │ │ │ ├── tweet_is_nsfw/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── TweetIsNsfwCandidateFeatureHydrator.scala │ │ │ │ ├── tweet_tlx/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ └── TweetTLXScoreCandidateFeatureHydrator.scala │ │ │ │ ├── tweet_tweetypie/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── TweetTweetypieCandidateFeatureHydrator.scala │ │ │ │ └── tweet_visibility_reason/ │ │ │ │ ├── BUILD │ │ │ │ └── TweetVisibilityReasonBulkCandidateFeatureHydrator.scala │ │ │ └── query/ │ │ │ ├── async/ │ │ │ │ ├── AsyncQueryFeatureHydrator.scala │ │ │ │ └── BUILD │ │ │ ├── cr_ml_ranker/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── CrMlRankerCommonQueryFeatureHydrator.scala │ │ │ │ ├── CrMlRankerCommonQueryFeatureHydratorBuilder.scala │ │ │ │ └── RankingConfigBuilder.scala │ │ │ ├── impressed_tweets/ │ │ │ │ ├── BUILD │ │ │ │ └── ImpressedTweetsQueryFeatureHydrator.scala │ │ │ ├── logged_in_only/ │ │ │ │ ├── BUILD │ │ │ │ └── LoggedInOnlyQueryFeatureHydrator.scala │ │ │ ├── param_gated/ │ │ │ │ ├── AsyncParamGatedQueryFeatureHydrator.scala │ │ │ │ ├── BUILD │ │ │ │ ├── ParamGatedQueryFeatureHydrator.scala │ │ │ │ └── featurestorev1/ │ │ │ │ ├── AsyncParamGatedFeatureStoreV1QueryFeatureHydrator.scala │ │ │ │ ├── BUILD │ │ │ │ └── ParamGatedFeatureStoreV1QueryFeatureHydrator.scala │ │ │ └── qualityfactor_gated/ │ │ │ ├── BUILD.bazel │ │ │ └── QualityFactorGatedQueryFeatureHydrator.scala │ │ ├── filter/ │ │ │ ├── AdaptiveLongIntBloomFilterDedupFilter.scala │ │ │ ├── BUILD │ │ │ ├── ExcludedIdsFilter.scala │ │ │ ├── FeatureFilter.scala │ │ │ ├── FeatureValueConditionalFilter.scala │ │ │ ├── HasAuthorIdFeatureFilter.scala │ │ │ ├── ParamGatedFilter.scala │ │ │ ├── PredicateFilter.scala │ │ │ ├── SnowflakeIdAgeFilter.scala │ │ │ ├── TweetAuthorCountryFilter.scala │ │ │ ├── TweetAuthorIsSelfFilter.scala │ │ │ ├── TweetIsNotReplyFilter.scala │ │ │ ├── TweetLanguageFilter.scala │ │ │ ├── TweetVisibilityFilter.scala │ │ │ ├── UrtUnorderedExcludeIdsCursorFilter.scala │ │ │ ├── list_visibility/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── ListVisibilityFilter.scala │ │ │ └── tweet_impression/ │ │ │ ├── BUILD.bazel │ │ │ └── TweetImpressionFilter.scala │ │ ├── gate/ │ │ │ ├── BUILD │ │ │ ├── DefinedCountryCodeGate.scala │ │ │ ├── FeatureGate.scala │ │ │ ├── FirstPageGate.scala │ │ │ ├── NoCandidatesGate.scala │ │ │ ├── NonEmptyAdsQueryStringGate.scala │ │ │ ├── NonEmptyCandidatesGate.scala │ │ │ ├── QualityFactorGate.scala │ │ │ └── any_candidates_without_feature/ │ │ │ ├── AnyCandidatesWithoutFeatureGate.scala │ │ │ └── BUILD.bazel │ │ ├── model/ │ │ │ ├── candidate/ │ │ │ │ ├── ArticleCandidate.scala │ │ │ │ ├── AudioSpaceCandidate.scala │ │ │ │ ├── BUILD │ │ │ │ ├── CardCandidate.scala │ │ │ │ ├── CommerceItemCandidate.scala │ │ │ │ ├── CursorCandidate.scala │ │ │ │ ├── DMConvoCandidate.scala │ │ │ │ ├── DMEventCandidate.scala │ │ │ │ ├── GenericSummaryCandidate.scala │ │ │ │ ├── LabelCandidate.scala │ │ │ │ ├── MomentAnnotationCandidate.scala │ │ │ │ ├── PromptCandidate.scala │ │ │ │ ├── ShowAlertCandidate.scala │ │ │ │ ├── TopicCandidate.scala │ │ │ │ ├── TweetCandidate.scala │ │ │ │ ├── TwitterListCandidate.scala │ │ │ │ ├── UserCandidate.scala │ │ │ │ ├── ads/ │ │ │ │ │ ├── AdsCandidate.scala │ │ │ │ │ └── BUILD │ │ │ │ ├── hubble/ │ │ │ │ │ ├── AdCreativeCandidate.scala │ │ │ │ │ ├── AdGroupCandidate.scala │ │ │ │ │ ├── AdUnitCandidate.scala │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── CampaignCandidate.scala │ │ │ │ │ └── FundingSourceCandidate.scala │ │ │ │ ├── suggestion/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── QuerySuggestionCandidate.scala │ │ │ │ │ └── SpellingSuggestionCandidate.scala │ │ │ │ └── trends_events/ │ │ │ │ ├── BUILD │ │ │ │ └── UnifiedTrendEventCandidate.scala │ │ │ ├── cursor/ │ │ │ │ ├── BUILD │ │ │ │ ├── OrderedCursor.scala │ │ │ │ ├── PassThroughCursor.scala │ │ │ │ ├── UnorderedBloomFilterCursor.scala │ │ │ │ ├── UnorderedExcludeIdsCursor.scala │ │ │ │ └── UrtPlaceholderCursor.scala │ │ │ ├── feature/ │ │ │ │ └── flexible_injection_pipeline/ │ │ │ │ └── BUILD.bazel │ │ │ ├── presentation/ │ │ │ │ ├── slice/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── SliceItemPresentation.scala │ │ │ │ └── urt/ │ │ │ │ ├── BUILD │ │ │ │ ├── ConversationModuleItem.scala │ │ │ │ ├── UrtItemPresentation.scala │ │ │ │ ├── UrtModulePresentation.scala │ │ │ │ └── UrtOperationPresentation.scala │ │ │ └── query/ │ │ │ └── ads/ │ │ │ ├── AdsQuery.scala │ │ │ └── BUILD │ │ ├── module/ │ │ │ ├── AccountRecommendationsMixerModule.scala │ │ │ ├── BUILD │ │ │ ├── ConversationServiceModule.scala │ │ │ ├── CrMixerClientModule.scala │ │ │ ├── DarkTrafficFilterModule.scala │ │ │ ├── EarlybirdModule.scala │ │ │ ├── ExploreRankerClientModule.scala │ │ │ ├── FollowRecommenderServiceModule.scala │ │ │ ├── GizmoduckClientModule.scala │ │ │ ├── HomeScorerClientModule.scala │ │ │ ├── InterestsDiscoveryServiceModule.scala │ │ │ ├── OnboardingTaskServiceModule.scala │ │ │ ├── PeopleDiscoveryServiceModule.scala │ │ │ ├── SocialGraphServiceModule.scala │ │ │ ├── TimelineMixerClientModule.scala │ │ │ ├── TimelineRankerClientModule.scala │ │ │ ├── TimelineScorerClientModule.scala │ │ │ ├── TimelineServiceClientModule.scala │ │ │ ├── TweetImpressionStoreModule.scala │ │ │ ├── TweetyPieClientModule.scala │ │ │ ├── UserSessionStoreModule.scala │ │ │ ├── cr_ml_ranker/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── CrMlRankerModule.scala │ │ │ └── http/ │ │ │ ├── BUILD │ │ │ ├── FinagleHttpClientModule.scala │ │ │ ├── FinagleHttpClientWithCredentialProxyModule.scala │ │ │ ├── FinagleHttpClientWithProxyModule.scala │ │ │ ├── FinatraHttpClientModule.scala │ │ │ ├── FinatraHttpClientWithCredentialProxyModule.scala │ │ │ ├── FinatraHttpClientWithProxyModule.scala │ │ │ └── ProxyCredentialsModule.scala │ │ ├── pipeline/ │ │ │ └── candidate/ │ │ │ ├── ads/ │ │ │ │ ├── AdsCandidatePipelineConfig.scala │ │ │ │ ├── AdsCandidatePipelineConfigBuilder.scala │ │ │ │ ├── AdsCandidatePipelineQueryTransformer.scala │ │ │ │ ├── AdsCandidatePipelineResultsTransformer.scala │ │ │ │ ├── AdsDependentCandidatePipelineConfig.scala │ │ │ │ ├── AdsDependentCandidatePipelineConfigBuilder.scala │ │ │ │ ├── AdsDependentCandidatePipelineQueryTransformer.scala │ │ │ │ ├── AdsDisplayLocationBuilder.scala │ │ │ │ ├── BUILD │ │ │ │ ├── CountNumOrganicItems.scala │ │ │ │ ├── GetOrganicItemIds.scala │ │ │ │ ├── PromotedTweetsOnlyFilter.scala │ │ │ │ └── ValidAdImpressionIdFilter.scala │ │ │ ├── flexible_injection_pipeline/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── FlipPromptCandidatePipelineConfig.scala │ │ │ │ ├── FlipPromptCandidatePipelineConfigBuilder.scala │ │ │ │ ├── FlipPromptDependentCandidatePipelineConfig.scala │ │ │ │ ├── FlipPromptDependentCandidatePipelineConfigBuilder.scala │ │ │ │ └── transformer/ │ │ │ │ ├── BUILD │ │ │ │ ├── FlipCandidateFeatureTransformer.scala │ │ │ │ ├── FlipInjectionParams.scala │ │ │ │ ├── FlipQueryTransformer.scala │ │ │ │ └── PromptResultsTransformer.scala │ │ │ └── who_to_follow_module/ │ │ │ ├── BUILD.bazel │ │ │ ├── WhoToFollowArmCandidateDecorator.scala │ │ │ ├── WhoToFollowArmCandidatePipelineConfig.scala │ │ │ ├── WhoToFollowArmCandidatePipelineQueryTransformer.scala │ │ │ ├── WhoToFollowArmDependentCandidatePipelineConfig.scala │ │ │ ├── WhoToFollowArmDependentCandidatePipelineConfigBuilder.scala │ │ │ ├── WhoToFollowArmResponseFeatureTransformer.scala │ │ │ ├── WhoToFollowCandidateDecorator.scala │ │ │ ├── WhoToFollowCandidatePipelineConfig.scala │ │ │ ├── WhoToFollowCandidatePipelineConfigBuilder.scala │ │ │ ├── WhoToFollowCandidatePipelineQueryTransformer.scala │ │ │ ├── WhoToFollowClientEventDetailsBuilder.scala │ │ │ ├── WhoToFollowDependentCandidatePipelineConfig.scala │ │ │ ├── WhoToFollowDependentCandidatePipelineConfigBuilder.scala │ │ │ └── WhoToFollowResponseFeatureTransformer.scala │ │ ├── premarshaller/ │ │ │ ├── cursor/ │ │ │ │ ├── BUILD │ │ │ │ ├── CursorSerializer.scala │ │ │ │ └── UrtCursorSerializer.scala │ │ │ ├── slice/ │ │ │ │ ├── BUILD │ │ │ │ ├── SliceDomainMarshaller.scala │ │ │ │ └── builder/ │ │ │ │ ├── BUILD │ │ │ │ ├── OrderedNextCursorBuilder.scala │ │ │ │ ├── OrderedNextCursorUpdater.scala │ │ │ │ ├── OrderedPreviousCursorBuilder.scala │ │ │ │ ├── OrderedPreviousCursorUpdater.scala │ │ │ │ ├── ShouldInclude.scala │ │ │ │ ├── SliceBuilder.scala │ │ │ │ ├── SliceCursorBuilder.scala │ │ │ │ └── SliceCursorUpdater.scala │ │ │ ├── urp/ │ │ │ │ ├── BUILD │ │ │ │ ├── UrpDomainMarshaller.scala │ │ │ │ └── builder/ │ │ │ │ ├── BUILD │ │ │ │ ├── PageBodyBuilder.scala │ │ │ │ ├── PageHeaderBuilder.scala │ │ │ │ ├── PageNavBarBuilder.scala │ │ │ │ ├── StaticTimelineScribeConfigBuilder.scala │ │ │ │ └── TimelineScribeConfigBuilder.scala │ │ │ └── urt/ │ │ │ ├── BUILD │ │ │ ├── UndecoratedUrtDomainMarshaller.scala │ │ │ ├── UrtDomainMarshaller.scala │ │ │ └── builder/ │ │ │ ├── AddEntriesInstructionBuilder.scala │ │ │ ├── AddEntriesWithAddToModuleInstructionBuilder.scala │ │ │ ├── AddEntriesWithPinnedAndReplaceInstructionBuilder.scala │ │ │ ├── AddEntriesWithReplaceAndShowAlertInstructionBuilder.scala │ │ │ ├── AddEntriesWithReplaceInstructionBuilder.scala │ │ │ ├── AddEntriesWithShowCoverInstructionBuilder.scala │ │ │ ├── AddToModuleInstructionBuilder.scala │ │ │ ├── BUILD │ │ │ ├── BaseUnorderedExcludeIdsBottomCursorBuilder.scala │ │ │ ├── ClearCacheInstructionBuilder.scala │ │ │ ├── FeaturePassThroughCursorBuilder.scala │ │ │ ├── IncludeInstruction.scala │ │ │ ├── MarkUnreadInstructionBuilder.scala │ │ │ ├── OrderedBottomCursorBuilder.scala │ │ │ ├── OrderedGapCursorBuilder.scala │ │ │ ├── OrderedTopCursorBuilder.scala │ │ │ ├── PinEntryInstructionBuilder.scala │ │ │ ├── PlaceholderTopCursorBuilder.scala │ │ │ ├── ReplaceEntryInstructionBuilder.scala │ │ │ ├── ShowAlertInstructionBuilder.scala │ │ │ ├── ShowCoverInstructionBuilder.scala │ │ │ ├── StaticTimelineScribeConfigBuilder.scala │ │ │ ├── TerminateInstructionBuilder.scala │ │ │ ├── TimelineScribeConfigBuilder.scala │ │ │ ├── UnorderedBloomFilterBottomCursorBuilder.scala │ │ │ ├── UnorderedExcludeIdsBottomCursorBuilder.scala │ │ │ ├── UnorderedExcludeIdsSeqBottomCursorBuilder.scala │ │ │ ├── UrtBuilder.scala │ │ │ ├── UrtCursorBuilder.scala │ │ │ ├── UrtCursorUpdater.scala │ │ │ ├── UrtInstructionBuilder.scala │ │ │ └── UrtMetadataBuilder.scala │ │ ├── scorer/ │ │ │ ├── common/ │ │ │ │ ├── BUILD │ │ │ │ ├── MLModelInferenceClient.scala │ │ │ │ ├── ManagedModelClient.scala │ │ │ │ ├── ModelSelector.scala │ │ │ │ └── NaviModelClient.scala │ │ │ ├── cortex/ │ │ │ │ ├── BUILD │ │ │ │ ├── CortexManagedInferenceServiceDataRecordScorer.scala │ │ │ │ ├── CortexManagedInferenceServiceDataRecordScorerBuilder.scala │ │ │ │ ├── CortexManagedInferenceServiceTensorScorer.scala │ │ │ │ ├── CortexManagedInferenceServiceTensorScorerBuilder.scala │ │ │ │ └── ModelFeatureExtractor.scala │ │ │ ├── cr_ml_ranker/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── CrMlRankerScorer.scala │ │ │ │ └── CrMlRankerStitchClient.scala │ │ │ ├── deepbird/ │ │ │ │ ├── BUILD │ │ │ │ ├── BaseDeepbirdV2Scorer.scala │ │ │ │ ├── DeepbirdV2PredictionServerScorer.scala │ │ │ │ ├── LollyPredictionEngineScorer.scala │ │ │ │ └── TensorflowPredictionEngineScorer.scala │ │ │ ├── param_gated/ │ │ │ │ ├── BUILD │ │ │ │ └── ParamGatedScorer.scala │ │ │ ├── qualityfactor_gated/ │ │ │ │ ├── BUILD │ │ │ │ └── QualityFactorGatedScorer.scala │ │ │ ├── tensorbuilder/ │ │ │ │ ├── BUILD │ │ │ │ ├── BooleanInferInputTensorBuilder.scala │ │ │ │ ├── BytesInferInputTensorBuilder.scala │ │ │ │ ├── CandidateInferInputTensorBuilder.scala │ │ │ │ ├── Float32InferInputTensorBuilder.scala │ │ │ │ ├── FloatTensorInferInputTensorBuilder.scala │ │ │ │ ├── InferInputTensorBuilder.scala │ │ │ │ ├── Int64InferInputTensorBuilder.scala │ │ │ │ ├── ModelInferRequestBuilder.scala │ │ │ │ ├── QueryInferInputTensorBuilder.scala │ │ │ │ └── SparseMapInferInputTensorBuilder.scala │ │ │ └── tweet_tlx/ │ │ │ ├── BUILD.bazel │ │ │ ├── TweetTLXStratoScorer.scala │ │ │ └── TweetTLXThriftScorer.scala │ │ ├── selector/ │ │ │ ├── BUILD │ │ │ ├── Bucketer.scala │ │ │ ├── CandidateMergeStrategy.scala │ │ │ ├── CandidatePositionInResults.scala │ │ │ ├── DeduplicationKey.scala │ │ │ ├── DropAllCandidates.scala │ │ │ ├── DropDuplicateCandidates.scala │ │ │ ├── DropDuplicateModuleItemCandidates.scala │ │ │ ├── DropDuplicateResults.scala │ │ │ ├── DropFilteredCandidates.scala │ │ │ ├── DropFilteredModuleItemCandidates.scala │ │ │ ├── DropMaxCandidates.scala │ │ │ ├── DropMaxModuleItemCandidates.scala │ │ │ ├── DropMaxResults.scala │ │ │ ├── DropModuleTooFewModuleItemResults.scala │ │ │ ├── DropNonDuplicateCandidates.scala │ │ │ ├── DropOrthogonalCandidates.scala │ │ │ ├── DropRequestedMaxModuleItemCandidates.scala │ │ │ ├── DropRequestedMaxResults.scala │ │ │ ├── DropSelector.scala │ │ │ ├── DropTooFewResults.scala │ │ │ ├── DynamicPositionSelector.scala │ │ │ ├── InsertAppendIntoModuleCandidates.scala │ │ │ ├── InsertAppendPatternResults.scala │ │ │ ├── InsertAppendRatioResults.scala │ │ │ ├── InsertAppendResults.scala │ │ │ ├── InsertAppendWeaveResults.scala │ │ │ ├── InsertAppendWithoutFeatureResults.scala │ │ │ ├── InsertDynamicPositionResults.scala │ │ │ ├── InsertFixedPositionIntoModuleCandidates.scala │ │ │ ├── InsertFixedPositionResults.scala │ │ │ ├── InsertIntoModule.scala │ │ │ ├── InsertPerCandidateDynamicPositionResults.scala │ │ │ ├── InsertRandomPositionResults.scala │ │ │ ├── InsertRelativePositionResults.scala │ │ │ ├── InsertSelector.scala │ │ │ ├── SelectConditionally.scala │ │ │ ├── SelectFromSubpoolCandidates.scala │ │ │ ├── UpdateSortCandidates.scala │ │ │ ├── UpdateSortModuleItemCandidates.scala │ │ │ ├── UpdateSortResults.scala │ │ │ ├── ads/ │ │ │ │ ├── AdsInjector.scala │ │ │ │ ├── BUILD.bazel │ │ │ │ └── InsertAdResults.scala │ │ │ └── sorter/ │ │ │ ├── BUILD │ │ │ ├── FeatureValueSorter.scala │ │ │ ├── RandomShuffleSorter.scala │ │ │ ├── ReverseSorter.scala │ │ │ ├── SortOrder.scala │ │ │ ├── SorterFromOrdering.scala │ │ │ ├── SorterProvider.scala │ │ │ └── featurestorev1/ │ │ │ ├── BUILD │ │ │ └── FeatureStoreV1FeatureValueSorter.scala │ │ └── side_effect/ │ │ ├── BUILD │ │ ├── KafkaPublishingSideEffect.scala │ │ ├── ParamGatedPipelineResultSideEffect.scala │ │ ├── ScribeClientEventSideEffect.scala │ │ ├── ScribeLogEventAsyncSideEffect.scala │ │ ├── ScribeLogEventSideEffect.scala │ │ ├── StratoInsertSideEffect.scala │ │ ├── UserSessionStoreUpdateSideEffect.scala │ │ └── metrics/ │ │ ├── BUILD.bazel │ │ ├── CandidateMetricFunction.scala │ │ ├── ScribeClientEventMetricsSideEffect.scala │ │ └── ScribeClientEventMetricsSideEffectBuilder.scala │ ├── core/ │ │ └── src/ │ │ └── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── twitter/ │ │ │ └── product_mixer/ │ │ │ └── core/ │ │ │ └── product/ │ │ │ └── guice/ │ │ │ └── scope/ │ │ │ ├── BUILD │ │ │ └── ProductScoped.java │ │ └── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── product_mixer/ │ │ └── core/ │ │ ├── controllers/ │ │ │ ├── AlertConfig.scala │ │ │ ├── BUILD │ │ │ ├── DebugTwitterContext.scala │ │ │ ├── GetComponentRegistryHandler.scala │ │ │ ├── GetDebugConfigurationHandler.scala │ │ │ ├── PredicateConfig.scala │ │ │ ├── ProductMixerController.scala │ │ │ └── QualityFactorMonitoringConfig.scala │ │ ├── feature/ │ │ │ ├── BUILD │ │ │ ├── Feature.scala │ │ │ ├── datarecord/ │ │ │ │ ├── BUILD │ │ │ │ ├── DataRecordCompatible.scala │ │ │ │ └── DataRecordFeature.scala │ │ │ ├── featuremap/ │ │ │ │ ├── BUILD │ │ │ │ ├── FeatureMap.scala │ │ │ │ ├── FeatureMapBuilder.scala │ │ │ │ ├── FeatureMapException.scala │ │ │ │ ├── FeatureMapSerializer.scala │ │ │ │ ├── asyncfeaturemap/ │ │ │ │ │ ├── AsyncFeatureMap.scala │ │ │ │ │ ├── AsyncFeatureMapSerializer.scala │ │ │ │ │ └── BUILD │ │ │ │ ├── datarecord/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── DataRecordConverter.scala │ │ │ │ │ ├── DataRecordExtractor.scala │ │ │ │ │ └── FeaturesScope.scala │ │ │ │ └── featurestorev1/ │ │ │ │ ├── BUILD │ │ │ │ └── FeatureStoreV1FeatureMap.scala │ │ │ └── featurestorev1/ │ │ │ ├── BUILD │ │ │ ├── FeatureStoreV1Entity.scala │ │ │ ├── FeatureStoreV1Feature.scala │ │ │ └── featurevalue/ │ │ │ ├── BUILD │ │ │ └── FeatureStoreV1Response.scala │ │ ├── functional_component/ │ │ │ ├── access_policy/ │ │ │ │ └── BUILD │ │ │ ├── candidate_source/ │ │ │ │ ├── BUILD │ │ │ │ ├── CandidateSource.scala │ │ │ │ ├── CandidatesWithSourceFeatures.scala │ │ │ │ ├── PassthroughCandidateSource.scala │ │ │ │ ├── StaticCandidateSource.scala │ │ │ │ ├── product_pipeline/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── ProductPipelineCandidateSource.scala │ │ │ │ └── strato/ │ │ │ │ ├── BUILD │ │ │ │ ├── StratoErrCategorizer.scala │ │ │ │ ├── StratoKeyFetcherSeqSource.scala │ │ │ │ ├── StratoKeyFetcherSource.scala │ │ │ │ ├── StratoKeyFetcherWithSourceFeaturesSource.scala │ │ │ │ ├── StratoKeyView.scala │ │ │ │ ├── StratoKeyViewFetcherSeqSource.scala │ │ │ │ ├── StratoKeyViewFetcherSource.scala │ │ │ │ └── StratoKeyViewFetcherWithSourceFeaturesSource.scala │ │ │ ├── common/ │ │ │ │ ├── BUILD │ │ │ │ ├── CandidateScope.scala │ │ │ │ ├── access_policy/ │ │ │ │ │ ├── AccessPolicy.scala │ │ │ │ │ ├── AccessPolicyEvaluator.scala │ │ │ │ │ ├── BUILD │ │ │ │ │ └── WithDebugAccessPolicies.scala │ │ │ │ └── alert/ │ │ │ │ ├── Alert.scala │ │ │ │ ├── AlertType.scala │ │ │ │ ├── BUILD │ │ │ │ ├── EmptyResponseRateAlert.scala │ │ │ │ ├── GenericClientLatencyAlert.scala │ │ │ │ ├── GenericClientSuccessRateAlert.scala │ │ │ │ ├── GenericClientThroughputAlert.scala │ │ │ │ ├── IsObservableFromStrato.scala │ │ │ │ ├── LatencyAlert.scala │ │ │ │ ├── NotificationGroup.scala │ │ │ │ ├── Percentile.scala │ │ │ │ ├── ResponseSizeAlert.scala │ │ │ │ ├── Source.scala │ │ │ │ ├── StratoColumnAlert.scala │ │ │ │ ├── SuccessRateAlert.scala │ │ │ │ ├── ThroughputAlert.scala │ │ │ │ └── predicate/ │ │ │ │ ├── BUILD │ │ │ │ ├── MetricGranularity.scala │ │ │ │ ├── Operator.scala │ │ │ │ ├── Predicate.scala │ │ │ │ ├── TriggerIfAbove.scala │ │ │ │ ├── TriggerIfBelow.scala │ │ │ │ └── TriggerIfLatencyAbove.scala │ │ │ ├── configapi/ │ │ │ │ ├── BUILD │ │ │ │ ├── ConfigBuilder.scala │ │ │ │ ├── ParamsBuilder.scala │ │ │ │ ├── RequestContext.scala │ │ │ │ ├── RequestContextBuilder.scala │ │ │ │ ├── StaticParam.scala │ │ │ │ └── registry/ │ │ │ │ ├── BUILD │ │ │ │ ├── GlobalParamConfig.scala │ │ │ │ ├── GlobalParamRegistry.scala │ │ │ │ ├── ParamConfig.scala │ │ │ │ └── ParamConfigBuilder.scala │ │ │ ├── decorator/ │ │ │ │ ├── BUILD │ │ │ │ ├── CandidateDecorator.scala │ │ │ │ ├── Decoration.scala │ │ │ │ ├── slice/ │ │ │ │ │ └── builder/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── CandidateSliceItemBuilder.scala │ │ │ │ └── urt/ │ │ │ │ └── builder/ │ │ │ │ ├── BUILD │ │ │ │ ├── CandidateUrtEntryBuilder.scala │ │ │ │ ├── icon/ │ │ │ │ │ └── BaseHorizonIconBuilder.scala │ │ │ │ ├── item/ │ │ │ │ │ ├── alert/ │ │ │ │ │ │ ├── BaseDurationBuilder.scala │ │ │ │ │ │ ├── BaseShowAlertColorConfigurationBuilder.scala │ │ │ │ │ │ ├── BaseShowAlertDisplayLocationBuilder.scala │ │ │ │ │ │ ├── BaseShowAlertIconDisplayInfoBuilder.scala │ │ │ │ │ │ ├── BaseShowAlertNavigationMetadataBuilder.scala │ │ │ │ │ │ └── BaseShowAlertUserIdsBuilder.scala │ │ │ │ │ ├── topic/ │ │ │ │ │ │ ├── BaseTopicDisplayTypeBuilder.scala │ │ │ │ │ │ └── BaseTopicFunctionalityTypeBuilder.scala │ │ │ │ │ ├── tweet/ │ │ │ │ │ │ ├── BaseEntryIdToReplaceBuilder.scala │ │ │ │ │ │ ├── BaseTimelinesScoreInfoBuilder.scala │ │ │ │ │ │ └── BaseTweetHighlightsBuilder.scala │ │ │ │ │ └── user/ │ │ │ │ │ └── BaseUserReactiveTriggersBuilder.scala │ │ │ │ ├── metadata/ │ │ │ │ │ ├── BaseClientEventDetailsBuilder.scala │ │ │ │ │ ├── BaseClientEventInfoBuilder.scala │ │ │ │ │ ├── BaseFeedbackActionInfoBuilder.scala │ │ │ │ │ ├── BaseModuleStr.scala │ │ │ │ │ ├── BaseStr.scala │ │ │ │ │ └── BaseUrlBuilder.scala │ │ │ │ ├── promoted/ │ │ │ │ │ └── BasePromotedMetadataBuilder.scala │ │ │ │ ├── richtext/ │ │ │ │ │ └── BaseRichTextBuilder.scala │ │ │ │ ├── social_context/ │ │ │ │ │ ├── BaseModuleSocialContextBuilder.scala │ │ │ │ │ └── BaseSocialContextBuilder.scala │ │ │ │ ├── stringcenter/ │ │ │ │ │ ├── BaseModuleStringCenterPlaceholderBuilder.scala │ │ │ │ │ └── BaseStringCenterPlaceholderBuilder.scala │ │ │ │ └── timeline_module/ │ │ │ │ ├── BaseModuleDisplayTypeBuilder.scala │ │ │ │ ├── BaseModuleFooterBuilder.scala │ │ │ │ ├── BaseModuleHeaderBuilder.scala │ │ │ │ ├── BaseModuleHeaderDisplayTypeBuilder.scala │ │ │ │ ├── BaseModuleMetadataBuilder.scala │ │ │ │ ├── BaseModuleShowMoreBehaviorBuilder.scala │ │ │ │ └── BaseTimelineModuleBuilder.scala │ │ │ ├── feature_hydrator/ │ │ │ │ ├── BUILD │ │ │ │ ├── CandidateFeatureHydrator.scala │ │ │ │ ├── FeatureHydrator.scala │ │ │ │ ├── HydratorCandidateResult.scala │ │ │ │ ├── QueryFeatureHydrator.scala │ │ │ │ └── featurestorev1/ │ │ │ │ ├── BUILD │ │ │ │ ├── FeatureStoreDatasetErrorHandler.scala │ │ │ │ ├── FeatureStoreV1CandidateFeatureHydrator.scala │ │ │ │ ├── FeatureStoreV1DynamicClientBuilder.scala │ │ │ │ ├── FeatureStoreV1HydrationConfig.scala │ │ │ │ └── FeatureStoreV1QueryFeatureHydrator.scala │ │ │ ├── filter/ │ │ │ │ ├── BUILD │ │ │ │ ├── Filter.scala │ │ │ │ └── FilterResult.scala │ │ │ ├── gate/ │ │ │ │ ├── BUILD │ │ │ │ ├── Gate.scala │ │ │ │ ├── GateResult.scala │ │ │ │ └── ShouldContinue.scala │ │ │ ├── marshaller/ │ │ │ │ ├── BUILD │ │ │ │ ├── TransportMarshaller.scala │ │ │ │ ├── request/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── ClientContextMarshaller.scala │ │ │ │ │ ├── ClientContextUnmarshaller.scala │ │ │ │ │ └── FeatureValueUnmarshaller.scala │ │ │ │ └── response/ │ │ │ │ ├── graphql/ │ │ │ │ │ └── contextual_ref/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── ContextualTweetRefMarshaller.scala │ │ │ │ │ ├── OuterTweetContextMarshaller.scala │ │ │ │ │ └── TweetHydrationContextMarshaller.scala │ │ │ │ ├── rtf/ │ │ │ │ │ └── safety_level/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── SafetyLevelMarshaller.scala │ │ │ │ ├── slice/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── CursorTypeMarshaller.scala │ │ │ │ │ ├── SliceItemMarshaller.scala │ │ │ │ │ └── SliceTransportMarshaller.scala │ │ │ │ ├── urp/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── PageBodyMarshaller.scala │ │ │ │ │ ├── PageHeaderMarshaller.scala │ │ │ │ │ ├── PageNavBarMarshaller.scala │ │ │ │ │ ├── SegmentedTimelineMarshaller.scala │ │ │ │ │ ├── SegmentedTimelinesMarshaller.scala │ │ │ │ │ ├── TimelineKeyMarshaller.scala │ │ │ │ │ ├── TitleNavBarMarshaller.scala │ │ │ │ │ ├── TopicPageHeaderDisplayTypeMarshaller.scala │ │ │ │ │ ├── TopicPageHeaderFacepileMarshaller.scala │ │ │ │ │ ├── TopicPageHeaderMarshaller.scala │ │ │ │ │ ├── TopicPageNavBarMarshaller.scala │ │ │ │ │ ├── UrpTransportMarshaller.scala │ │ │ │ │ └── UrpTransportMarshallerBuilder.scala │ │ │ │ └── urt/ │ │ │ │ ├── AddEntriesInstructionMarshaller.scala │ │ │ │ ├── AddToModuleInstructionMarshaller.scala │ │ │ │ ├── BUILD │ │ │ │ ├── CoverMarshaller.scala │ │ │ │ ├── MarkEntriesUnreadInstructionMarshaller.scala │ │ │ │ ├── ModuleItemMarshaller.scala │ │ │ │ ├── ModuleItemTreeDisplayMarshaller.scala │ │ │ │ ├── PinEntryInstructionMarshaller.scala │ │ │ │ ├── ReaderModeConfigMarshaller.scala │ │ │ │ ├── ReplaceEntryInstructionMarshaller.scala │ │ │ │ ├── ShowAlertInstructionMarshaller.scala │ │ │ │ ├── TerminateTimelineInstructionMarshaller.scala │ │ │ │ ├── TimelineEntryContentMarshaller.scala │ │ │ │ ├── TimelineEntryMarshaller.scala │ │ │ │ ├── TimelineInstructionMarshaller.scala │ │ │ │ ├── TimelineItemContentMarshaller.scala │ │ │ │ ├── TimelineItemMarshaller.scala │ │ │ │ ├── TimelineMetadataMarshaller.scala │ │ │ │ ├── TimelineModuleMarshaller.scala │ │ │ │ ├── TimelineOperationMarshaller.scala │ │ │ │ ├── TimelineScribeConfigMarshaller.scala │ │ │ │ ├── UrtTransportMarshaller.scala │ │ │ │ ├── UrtTransportMarshallerBuilder.scala │ │ │ │ ├── alert/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── ShowAlertColorConfigurationMarshaller.scala │ │ │ │ │ ├── ShowAlertDisplayLocationMarshaller.scala │ │ │ │ │ ├── ShowAlertIconDisplayInfoMarshaller.scala │ │ │ │ │ ├── ShowAlertIconMarshaller.scala │ │ │ │ │ ├── ShowAlertNavigationMetadataMarshaller.scala │ │ │ │ │ └── ShowAlertTypeMarshaller.scala │ │ │ │ ├── button/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── ButtonStyleMarshaller.scala │ │ │ │ │ ├── CtaButtonMarshaller.scala │ │ │ │ │ ├── IconCtaButtonMarshaller.scala │ │ │ │ │ └── TextCtaButtonMarshaller.scala │ │ │ │ ├── color/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── ColorMarshaller.scala │ │ │ │ │ ├── ColorPaletteMarshaller.scala │ │ │ │ │ └── RosettaColorMarshaller.scala │ │ │ │ ├── cover/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── CoverContentMarshaller.scala │ │ │ │ │ ├── CoverCtaBehaviorMarshaller.scala │ │ │ │ │ ├── CoverCtaMarshaller.scala │ │ │ │ │ ├── CoverImageMarshaller.scala │ │ │ │ │ ├── FullCoverContentMarshaller.scala │ │ │ │ │ ├── FullCoverDisplayTypeMarshaller.scala │ │ │ │ │ ├── HalfCoverContentMarshaller.scala │ │ │ │ │ └── HalfCoverDisplayTypeMarshaller.scala │ │ │ │ ├── icon/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── HorizonIconMarshaller.scala │ │ │ │ ├── item/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── article/ │ │ │ │ │ │ ├── ArticleDisplayTypeMarshaller.scala │ │ │ │ │ │ ├── ArticleItemMarshaller.scala │ │ │ │ │ │ └── ArticleSeedTypeMarshaller.scala │ │ │ │ │ ├── audio_space/ │ │ │ │ │ │ └── AudioSpaceItemMarshaller.scala │ │ │ │ │ ├── card/ │ │ │ │ │ │ ├── CardDisplayTypeMarshaller.scala │ │ │ │ │ │ └── CardItemMarshaller.scala │ │ │ │ │ ├── commerce/ │ │ │ │ │ │ ├── CommerceProductGroupItemMarshaller.scala │ │ │ │ │ │ └── CommerceProductItemMarshaller.scala │ │ │ │ │ ├── conversation_annotation/ │ │ │ │ │ │ ├── ConversationAnnotationMarshaller.scala │ │ │ │ │ │ └── ConversationAnnotationTypeMarshaller.scala │ │ │ │ │ ├── event/ │ │ │ │ │ │ ├── EventSummaryDisplayTypeMarshaller.scala │ │ │ │ │ │ └── EventSummaryItemMarshaller.scala │ │ │ │ │ ├── forward_pivot/ │ │ │ │ │ │ ├── ForwardPivotDisplayTypeMarshaller.scala │ │ │ │ │ │ ├── ForwardPivotMarshaller.scala │ │ │ │ │ │ └── SoftInterventionDisplayTypeMarshaller.scala │ │ │ │ │ ├── generic_summary_item/ │ │ │ │ │ │ ├── GenericSummaryActionMarshaller.scala │ │ │ │ │ │ ├── GenericSummaryContextMarshaller.scala │ │ │ │ │ │ ├── GenericSummaryDisplayTypeMarshaller.scala │ │ │ │ │ │ └── GenericSummaryItemMarshaller.scala │ │ │ │ │ ├── highlight/ │ │ │ │ │ │ └── HighlightedSectionMarshaller.scala │ │ │ │ │ ├── icon_label/ │ │ │ │ │ │ └── IconLabelItemMarshaller.scala │ │ │ │ │ ├── label/ │ │ │ │ │ │ ├── LabelDisplayTypeMarshaller.scala │ │ │ │ │ │ └── LabelItemMarshaller.scala │ │ │ │ │ ├── message/ │ │ │ │ │ │ ├── CompactPromptMessageContentMarshaller.scala │ │ │ │ │ │ ├── HeaderImagePromptMessageContentMarshaller.scala │ │ │ │ │ │ ├── InlinePromptMessageContentMarshaller.scala │ │ │ │ │ │ ├── MessageActionMarshaller.scala │ │ │ │ │ │ ├── MessageActionTypeMarshaller.scala │ │ │ │ │ │ ├── MessageContentMarshaller.scala │ │ │ │ │ │ ├── MessageImageMarshaller.scala │ │ │ │ │ │ ├── MessagePromptItemMarshaller.scala │ │ │ │ │ │ ├── MessageTextActionMarshaller.scala │ │ │ │ │ │ ├── UserFacepileDisplayTypeMarshaller.scala │ │ │ │ │ │ └── UserFacepileMarshaller.scala │ │ │ │ │ ├── moment/ │ │ │ │ │ │ └── MomentAnnotationItemMarshaller.scala │ │ │ │ │ ├── prompt/ │ │ │ │ │ │ ├── PromptContentMarshaller.scala │ │ │ │ │ │ ├── PromptItemMarshaller.scala │ │ │ │ │ │ ├── RelevancePromptContentMarshaller.scala │ │ │ │ │ │ ├── RelevancePromptDisplayTypeMarshaller.scala │ │ │ │ │ │ ├── RelevancePromptFollowUpFeedbackTypeMarshaller.scala │ │ │ │ │ │ └── RelevancePromptFollowUpTextInputMarshaller.scala │ │ │ │ │ ├── suggestion/ │ │ │ │ │ │ ├── SpellingActionTypeMarshaller.scala │ │ │ │ │ │ ├── SpellingItemMarshaller.scala │ │ │ │ │ │ └── TextResultMarshaller.scala │ │ │ │ │ ├── thread/ │ │ │ │ │ │ ├── ThreadHeaderContentMarshaller.scala │ │ │ │ │ │ └── ThreadHeaderItemMarshaller.scala │ │ │ │ │ ├── tile/ │ │ │ │ │ │ ├── CallToActionTileContentMarshaller.scala │ │ │ │ │ │ ├── StandardTileContentMarshaller.scala │ │ │ │ │ │ ├── TileContentMarshaller.scala │ │ │ │ │ │ └── TileItemMarshaller.scala │ │ │ │ │ ├── tombstone/ │ │ │ │ │ │ ├── TombstoneDisplayTypeMarshaller.scala │ │ │ │ │ │ ├── TombstoneInfoMarshaller.scala │ │ │ │ │ │ └── TombstoneItemMarshaller.scala │ │ │ │ │ ├── topic/ │ │ │ │ │ │ ├── TopicDisplayTypeMarshaller.scala │ │ │ │ │ │ ├── TopicFollowPromptDisplayTypeMarshaller.scala │ │ │ │ │ │ ├── TopicFollowPromptItemMarshaller.scala │ │ │ │ │ │ ├── TopicFunctionalityTypeMarshaller.scala │ │ │ │ │ │ └── TopicItemMarshaller.scala │ │ │ │ │ ├── trend/ │ │ │ │ │ │ └── TrendItemMarshaller.scala │ │ │ │ │ ├── tweet/ │ │ │ │ │ │ ├── TimelinesScoreInfoMarshaller.scala │ │ │ │ │ │ ├── TweetDisplayTypeMarshaller.scala │ │ │ │ │ │ ├── TweetHighlightsMarshaller.scala │ │ │ │ │ │ └── TweetItemMarshaller.scala │ │ │ │ │ ├── tweet_composer/ │ │ │ │ │ │ ├── TweetComposerDisplayTypeMarshaller.scala │ │ │ │ │ │ └── TweetComposerItemMarshaller.scala │ │ │ │ │ ├── twitter_list/ │ │ │ │ │ │ ├── TwitterListDisplayTypeMarshaller.scala │ │ │ │ │ │ └── TwitterListItemMarshaller.scala │ │ │ │ │ ├── user/ │ │ │ │ │ │ ├── UserDisplayTypeMarshaller.scala │ │ │ │ │ │ ├── UserItemMarshaller.scala │ │ │ │ │ │ └── UserReactiveTriggersMarshaller.scala │ │ │ │ │ └── vertical_grid_item/ │ │ │ │ │ ├── VerticalGridItemContentMarshaller.scala │ │ │ │ │ ├── VerticalGridItemMarshaller.scala │ │ │ │ │ ├── VerticalGridItemTileStyleMarshaller.scala │ │ │ │ │ ├── VerticalGridItemTopicFunctionalityTypeMarshaller.scala │ │ │ │ │ └── VerticalGridItemTopicTileMarshaller.scala │ │ │ │ ├── media/ │ │ │ │ │ ├── AspectRatioMarshaller.scala │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── BroadcastIdMarshaller.scala │ │ │ │ │ ├── MediaEntityMarshaller.scala │ │ │ │ │ ├── MediaKeyMarshaller.scala │ │ │ │ │ ├── MediaMarshaller.scala │ │ │ │ │ ├── RectMarshaller.scala │ │ │ │ │ └── TweetMediaMarshaller.scala │ │ │ │ ├── metadata/ │ │ │ │ │ ├── ArticleDetailsMarshaller.scala │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── BadgeMarshaller.scala │ │ │ │ │ ├── CallbackMarshaller.scala │ │ │ │ │ ├── ChildFeedbackActionMarshaller.scala │ │ │ │ │ ├── ClientEventDetailsMarshaller.scala │ │ │ │ │ ├── ClientEventInfoMarshaller.scala │ │ │ │ │ ├── CommerceDetailsMarshaller.scala │ │ │ │ │ ├── ConfirmationDisplayTypeMarshaller.scala │ │ │ │ │ ├── ConversationDetailsMarshaller.scala │ │ │ │ │ ├── ConversationSectionMarshaller.scala │ │ │ │ │ ├── DismissInfoMarshaller.scala │ │ │ │ │ ├── FeedbackActionMarshaller.scala │ │ │ │ │ ├── FeedbackDisplayContextMarshaller.scala │ │ │ │ │ ├── FeedbackInfoMarshaller.scala │ │ │ │ │ ├── FeedbackTypeMarshaller.scala │ │ │ │ │ ├── GeneralContextMarshaller.scala │ │ │ │ │ ├── GeneralContextTypeMarshaller.scala │ │ │ │ │ ├── ImageAnimationTypeMarshaller.scala │ │ │ │ │ ├── ImageDisplayTypeMarshaller.scala │ │ │ │ │ ├── ImageVariantMarshaller.scala │ │ │ │ │ ├── LiveEventDetailsMarshaller.scala │ │ │ │ │ ├── RichFeedbackBehaviorMarshaller.scala │ │ │ │ │ ├── SocialContextMarshaller.scala │ │ │ │ │ ├── TimelinesDetailsMarshaller.scala │ │ │ │ │ ├── TopicContextFunctionalityTypeMarshaller.scala │ │ │ │ │ ├── TopicContextMarshaller.scala │ │ │ │ │ ├── UrlMarshaller.scala │ │ │ │ │ ├── UrlTypeMarshaller.scala │ │ │ │ │ └── UrtEndpointOptionsMarshaller.scala │ │ │ │ ├── operation/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── CursorDisplayTreatmentMarshaller.scala │ │ │ │ │ ├── CursorItemMarshaller.scala │ │ │ │ │ ├── CursorOperationMarshaller.scala │ │ │ │ │ └── CursorTypeMarshaller.scala │ │ │ │ ├── promoted/ │ │ │ │ │ ├── AdMetadataContainerMarshaller.scala │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── CallToActionMarshaller.scala │ │ │ │ │ ├── ClickTrackingInfoMarshaller.scala │ │ │ │ │ ├── DisclaimerTypeMarshaller.scala │ │ │ │ │ ├── DisclosureTypeMarshaller.scala │ │ │ │ │ ├── DynamicPrerollTypeMarshaller.scala │ │ │ │ │ ├── MediaInfoMarshaller.scala │ │ │ │ │ ├── PrerollMarshaller.scala │ │ │ │ │ ├── PrerollMetadataMarshaller.scala │ │ │ │ │ ├── PromotedMetadataMarshaller.scala │ │ │ │ │ ├── SkAdNetworkDataMarshaller.scala │ │ │ │ │ ├── SponsorshipTypeMarshaller.scala │ │ │ │ │ ├── UrlOverrideTypeMarshaller.scala │ │ │ │ │ └── VideoVariantsMarshaller.scala │ │ │ │ ├── reaction/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── TimelineReactionMarshaller.scala │ │ │ │ ├── richtext/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── ReferenceObjectMarshaller.scala │ │ │ │ │ ├── RichTextAlignmentMarshaller.scala │ │ │ │ │ ├── RichTextEntityMarshaller.scala │ │ │ │ │ ├── RichTextFormatMarshaller.scala │ │ │ │ │ └── RichTextMarshaller.scala │ │ │ │ └── timeline_module/ │ │ │ │ ├── AdsMetadataMarshaller.scala │ │ │ │ ├── BUILD │ │ │ │ ├── GridCarouselMetadataMarshaller.scala │ │ │ │ ├── ModuleConversationMetadataMarshaller.scala │ │ │ │ ├── ModuleDisplayTypeMarshaller.scala │ │ │ │ ├── ModuleFooterMarshaller.scala │ │ │ │ ├── ModuleHeaderDisplayTypeMarshaller.scala │ │ │ │ ├── ModuleHeaderMarshaller.scala │ │ │ │ ├── ModuleMetadataMarshaller.scala │ │ │ │ ├── ModuleShowMoreBehaviorMarshaller.scala │ │ │ │ └── ModuleShowMoreBehaviorRevealByCountMarshaller.scala │ │ │ ├── premarshaller/ │ │ │ │ ├── BUILD │ │ │ │ └── DomainMarshaller.scala │ │ │ ├── scorer/ │ │ │ │ ├── BUILD │ │ │ │ ├── ScoredCandidateResult.scala │ │ │ │ └── Scorer.scala │ │ │ ├── selector/ │ │ │ │ ├── BUILD │ │ │ │ ├── Selector.scala │ │ │ │ └── SelectorResult.scala │ │ │ ├── side_effect/ │ │ │ │ ├── BUILD │ │ │ │ ├── ExecuteSynchronously.scala │ │ │ │ ├── PipelineResultSideEffect.scala │ │ │ │ └── SideEffect.scala │ │ │ └── transformer/ │ │ │ ├── BUILD │ │ │ ├── CandidateFeatureTransformer.scala │ │ │ ├── CandidatePipelineQueryTransformer.scala │ │ │ ├── CandidatePipelineResultsTransformer.scala │ │ │ ├── FeatureTransformer.scala │ │ │ └── Transformer.scala │ │ ├── gate/ │ │ │ ├── BUILD │ │ │ ├── DenyLoggedOutUsersGate.scala │ │ │ ├── ParamGate.scala │ │ │ └── ParamNotGate.scala │ │ ├── model/ │ │ │ ├── common/ │ │ │ │ ├── BUILD │ │ │ │ ├── CandidateWithFeatures.scala │ │ │ │ ├── Component.scala │ │ │ │ ├── Conditionally.scala │ │ │ │ ├── UniversalNoun.scala │ │ │ │ ├── identifier/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── CandidatePipelineIdentifier.scala │ │ │ │ │ ├── CandidateSourceIdentifier.scala │ │ │ │ │ ├── ComponentIdentifier.scala │ │ │ │ │ ├── ComponentIdentifierSerializer.scala │ │ │ │ │ ├── ComponentIdentifierStack.scala │ │ │ │ │ ├── ComponentIdentifierStackSerializer.scala │ │ │ │ │ ├── DecoratorIdentifier.scala │ │ │ │ │ ├── DomainMarshallerIdentifier.scala │ │ │ │ │ ├── FeatureHydratorIdentifier.scala │ │ │ │ │ ├── FilterIdentifier.scala │ │ │ │ │ ├── GateIdentifier.scala │ │ │ │ │ ├── MixerPipelineIdentifier.scala │ │ │ │ │ ├── PipelineStepIdentifier.scala │ │ │ │ │ ├── PlatformIdentifier.scala │ │ │ │ │ ├── ProductIdentifier.scala │ │ │ │ │ ├── ProductPipelineIdentifier.scala │ │ │ │ │ ├── RecommendationPipelineIdentifier.scala │ │ │ │ │ ├── RootIdentifier.scala │ │ │ │ │ ├── ScorerIdentifier.scala │ │ │ │ │ ├── ScoringPipelineIdentifier.scala │ │ │ │ │ ├── SelectorIdentifier.scala │ │ │ │ │ ├── SideEffectIdentifier.scala │ │ │ │ │ ├── TransformerIdentifier.scala │ │ │ │ │ └── TransportMarshallerIdentifier.scala │ │ │ │ └── presentation/ │ │ │ │ ├── BUILD │ │ │ │ ├── CandidateFeatures.scala │ │ │ │ ├── CandidateWithDetails.scala │ │ │ │ ├── ItemPresentation.scala │ │ │ │ ├── ModulePresentation.scala │ │ │ │ ├── UniversalPresentation.scala │ │ │ │ ├── slice/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── BaseSliceItemPresentation.scala │ │ │ │ └── urt/ │ │ │ │ ├── BUILD │ │ │ │ ├── BaseUrtItemPresentation.scala │ │ │ │ ├── BaseUrtModulePresentation.scala │ │ │ │ ├── BaseUrtOperationPresentation.scala │ │ │ │ ├── IsDispensable.scala │ │ │ │ └── WithItemTreeDisplay.scala │ │ │ └── marshalling/ │ │ │ ├── BUILD │ │ │ ├── HasMarshalling.scala │ │ │ ├── request/ │ │ │ │ ├── BUILD │ │ │ │ ├── ClientContext.scala │ │ │ │ ├── DebugOptions.scala │ │ │ │ ├── DebugParams.scala │ │ │ │ ├── HasExcludedIds.scala │ │ │ │ ├── HasSerializedRequestCursor.scala │ │ │ │ ├── Product.scala │ │ │ │ ├── ProductContext.scala │ │ │ │ └── Request.scala │ │ │ └── response/ │ │ │ ├── rtf/ │ │ │ │ └── safety_level/ │ │ │ │ ├── BUILD │ │ │ │ └── SafetyLevel.scala │ │ │ ├── slice/ │ │ │ │ ├── BUILD │ │ │ │ └── SliceItem.scala │ │ │ ├── urp/ │ │ │ │ ├── BUILD │ │ │ │ ├── Page.scala │ │ │ │ ├── PageBody.scala │ │ │ │ ├── PageHeader.scala │ │ │ │ ├── PageNavBar.scala │ │ │ │ ├── SegmentedTimeline.scala │ │ │ │ ├── TimelineKey.scala │ │ │ │ ├── TopicPageHeaderDisplayType.scala │ │ │ │ └── TopicPageHeaderFacepile.scala │ │ │ └── urt/ │ │ │ ├── BUILD │ │ │ ├── Cover.scala │ │ │ ├── EntryNamespace.scala │ │ │ ├── HasEntryIdentifier.scala │ │ │ ├── HasExpirationTime.scala │ │ │ ├── HasSortIndex.scala │ │ │ ├── ModuleItemTreeDisplay.scala │ │ │ ├── ReaderModeConfig.scala │ │ │ ├── ShowAlert.scala │ │ │ ├── Timeline.scala │ │ │ ├── TimelineEntry.scala │ │ │ ├── TimelineInstruction.scala │ │ │ ├── TimelineMetadata.scala │ │ │ ├── TimelineScribeConfig.scala │ │ │ ├── alert/ │ │ │ │ ├── BUILD │ │ │ │ ├── ShowAlertColorConfiguration.scala │ │ │ │ ├── ShowAlertDisplayLocation.scala │ │ │ │ ├── ShowAlertIcon.scala │ │ │ │ ├── ShowAlertIconDisplayInfo.scala │ │ │ │ ├── ShowAlertNavigationMetadata.scala │ │ │ │ └── ShowAlertType.scala │ │ │ ├── button/ │ │ │ │ ├── BUILD │ │ │ │ ├── ButtonStyle.scala │ │ │ │ └── CtaButton.scala │ │ │ ├── color/ │ │ │ │ ├── BUILD │ │ │ │ ├── Color.scala │ │ │ │ ├── ColorPalette.scala │ │ │ │ └── RosettaColor.scala │ │ │ ├── contextual_ref/ │ │ │ │ ├── BUILD │ │ │ │ ├── ContextualTweetRef.scala │ │ │ │ ├── OuterTweetContext.scala │ │ │ │ └── TweetHydrationContext.scala │ │ │ ├── cover/ │ │ │ │ ├── BUILD │ │ │ │ ├── CoverContent.scala │ │ │ │ ├── CoverCta.scala │ │ │ │ ├── CoverCtaBehavior.scala │ │ │ │ ├── CoverImage.scala │ │ │ │ ├── FullCoverDisplayType.scala │ │ │ │ ├── HalfCoverDisplayType.scala │ │ │ │ └── ShowCover.scala │ │ │ ├── icon/ │ │ │ │ ├── BUILD │ │ │ │ └── HorizonIcon.scala │ │ │ ├── item/ │ │ │ │ ├── BUILD │ │ │ │ ├── article/ │ │ │ │ │ ├── ArticleDisplayType.scala │ │ │ │ │ ├── ArticleItem.scala │ │ │ │ │ └── ArticleSeedType.scala │ │ │ │ ├── audio_space/ │ │ │ │ │ └── AudioSpaceItem.scala │ │ │ │ ├── card/ │ │ │ │ │ ├── CardDisplayType.scala │ │ │ │ │ └── CardItem.scala │ │ │ │ ├── commerce/ │ │ │ │ │ ├── CommerceProductGroupItem.scala │ │ │ │ │ └── CommerceProductItem.scala │ │ │ │ ├── conversation_annotation/ │ │ │ │ │ ├── ConversationAnnotation.scala │ │ │ │ │ └── ConversationAnnotationType.scala │ │ │ │ ├── event/ │ │ │ │ │ ├── EventSummaryDisplayType.scala │ │ │ │ │ └── EventSummaryItem.scala │ │ │ │ ├── forward_pivot/ │ │ │ │ │ ├── ForwardPivot.scala │ │ │ │ │ ├── ForwardPivotDisplayType.scala │ │ │ │ │ └── SoftInterventionDisplayType.scala │ │ │ │ ├── generic_summary/ │ │ │ │ │ ├── GenericSummaryAction.scala │ │ │ │ │ ├── GenericSummaryContext.scala │ │ │ │ │ ├── GenericSummaryDisplayType.scala │ │ │ │ │ └── GenericSummaryItem.scala │ │ │ │ ├── highlight/ │ │ │ │ │ └── HighlightedSection.scala │ │ │ │ ├── icon_label/ │ │ │ │ │ └── IconLabelItem.scala │ │ │ │ ├── label/ │ │ │ │ │ ├── LabelDisplayType.scala │ │ │ │ │ └── LabelItem.scala │ │ │ │ ├── message/ │ │ │ │ │ ├── MessageAction.scala │ │ │ │ │ ├── MessageActionType.scala │ │ │ │ │ ├── MessageContent.scala │ │ │ │ │ ├── MessageImage.scala │ │ │ │ │ ├── MessagePromptItem.scala │ │ │ │ │ ├── MessageTextAction.scala │ │ │ │ │ ├── UserFacepile.scala │ │ │ │ │ └── UserFacepileDisplayType.scala │ │ │ │ ├── moment/ │ │ │ │ │ └── MomentAnnotationItem.scala │ │ │ │ ├── prompt/ │ │ │ │ │ ├── PromptContent.scala │ │ │ │ │ ├── PromptItem.scala │ │ │ │ │ ├── RelevancePromptDisplayType.scala │ │ │ │ │ └── RelevancePromptFollowUpFeedbackType.scala │ │ │ │ ├── suggestion/ │ │ │ │ │ ├── SpellingActionType.scala │ │ │ │ │ ├── SpellingItem.scala │ │ │ │ │ └── TextResult.scala │ │ │ │ ├── thread/ │ │ │ │ │ ├── ThreadHeaderContent.scala │ │ │ │ │ └── ThreadHeaderItem.scala │ │ │ │ ├── tile/ │ │ │ │ │ ├── TileContent.scala │ │ │ │ │ └── TileItem.scala │ │ │ │ ├── tombstone/ │ │ │ │ │ ├── TombstoneDisplayType.scala │ │ │ │ │ ├── TombstoneInfo.scala │ │ │ │ │ └── TombstoneItem.scala │ │ │ │ ├── topic/ │ │ │ │ │ ├── TopicDisplayType.scala │ │ │ │ │ ├── TopicFollowPromptDisplayType.scala │ │ │ │ │ ├── TopicFollowPromptItem.scala │ │ │ │ │ ├── TopicFunctionalityType.scala │ │ │ │ │ └── TopicItem.scala │ │ │ │ ├── trend/ │ │ │ │ │ └── TrendItem.scala │ │ │ │ ├── tweet/ │ │ │ │ │ ├── TimelinesScoreInfo.scala │ │ │ │ │ ├── TweetDisplayType.scala │ │ │ │ │ ├── TweetHighlights.scala │ │ │ │ │ └── TweetItem.scala │ │ │ │ ├── tweet_composer/ │ │ │ │ │ ├── TweetComposerDisplayType.scala │ │ │ │ │ └── TweetComposerItem.scala │ │ │ │ ├── twitter_list/ │ │ │ │ │ ├── TwitterListDisplayType.scala │ │ │ │ │ └── TwitterListItem.scala │ │ │ │ ├── user/ │ │ │ │ │ ├── UserDisplayType.scala │ │ │ │ │ ├── UserItem.scala │ │ │ │ │ └── UserReactiveTriggers.scala │ │ │ │ └── vertical_grid_item/ │ │ │ │ ├── VerticalGridItem.scala │ │ │ │ ├── VerticalGridItemTileStyle.scala │ │ │ │ └── VerticalGridItemTopicFunctionalityType.scala │ │ │ ├── media/ │ │ │ │ ├── AspectRatio.scala │ │ │ │ ├── BUILD │ │ │ │ ├── Media.scala │ │ │ │ ├── MediaEntity.scala │ │ │ │ ├── MediaKey.scala │ │ │ │ └── Rect.scala │ │ │ ├── metadata/ │ │ │ │ ├── ArticleDetails.scala │ │ │ │ ├── BUILD │ │ │ │ ├── Badge.scala │ │ │ │ ├── Callback.scala │ │ │ │ ├── ClientEventInfo.scala │ │ │ │ ├── CommerceDetails.scala │ │ │ │ ├── ConfirmationDisplayType.scala │ │ │ │ ├── ConversationDetails.scala │ │ │ │ ├── ConversationSection.scala │ │ │ │ ├── DismissInfo.scala │ │ │ │ ├── FeedbackAction.scala │ │ │ │ ├── FeedbackActionInfo.scala │ │ │ │ ├── FeedbackInfo.scala │ │ │ │ ├── FeedbackType.scala │ │ │ │ ├── ImageAnimationType.scala │ │ │ │ ├── ImageDisplayType.scala │ │ │ │ ├── ImageVariant.scala │ │ │ │ ├── LiveEventDetails.scala │ │ │ │ ├── MarkUnreadableEntry.scala │ │ │ │ ├── PinnableEntry.scala │ │ │ │ ├── ReplaceableEntry.scala │ │ │ │ ├── ReplyPinState.scala │ │ │ │ ├── RichFeedbackBehavior.scala │ │ │ │ ├── SocialContext.scala │ │ │ │ ├── TimelinesDetails.scala │ │ │ │ └── Url.scala │ │ │ ├── operation/ │ │ │ │ ├── BUILD │ │ │ │ ├── CursorDisplayTreatment.scala │ │ │ │ ├── CursorItem.scala │ │ │ │ ├── CursorOperation.scala │ │ │ │ └── CursorType.scala │ │ │ ├── promoted/ │ │ │ │ ├── AdMetadataContainer.scala │ │ │ │ ├── BUILD │ │ │ │ ├── CallToAction.scala │ │ │ │ ├── ClickTrackingInfo.scala │ │ │ │ ├── DisclaimerType.scala │ │ │ │ ├── DisclosureType.scala │ │ │ │ ├── DynamicPrerollType.scala │ │ │ │ ├── MediaInfo.scala │ │ │ │ ├── Preroll.scala │ │ │ │ ├── PrerollMetadata.scala │ │ │ │ ├── PromotedMetadata.scala │ │ │ │ ├── SkAdNetworkData.scala │ │ │ │ ├── SponsorshipType.scala │ │ │ │ ├── UrlOverrideType.scala │ │ │ │ └── VideoVariant.scala │ │ │ ├── reaction/ │ │ │ │ ├── BUILD │ │ │ │ ├── TimelineReaction.scala │ │ │ │ └── TimelineReactionExecution.scala │ │ │ ├── richtext/ │ │ │ │ ├── BUILD │ │ │ │ ├── ReferenceObject.scala │ │ │ │ ├── RichText.scala │ │ │ │ ├── RichTextAlignment.scala │ │ │ │ ├── RichTextEntity.scala │ │ │ │ └── RichTextFormat.scala │ │ │ └── timeline_module/ │ │ │ ├── AdsMetadata.scala │ │ │ ├── BUILD │ │ │ ├── GridCarouselMetadata.scala │ │ │ ├── ModuleConversationMetadata.scala │ │ │ ├── ModuleDisplayType.scala │ │ │ ├── ModuleFooter.scala │ │ │ ├── ModuleHeader.scala │ │ │ ├── ModuleHeaderDisplayType.scala │ │ │ ├── ModuleMetadata.scala │ │ │ └── ModuleShowMoreBehavior.scala │ │ ├── module/ │ │ │ ├── ABDeciderModule.scala │ │ │ ├── BUILD │ │ │ ├── ConfigApiModule.scala │ │ │ ├── FeatureSwitchesModule.scala │ │ │ ├── LoggingThrowableExceptionMapper.scala │ │ │ ├── PipelineExecutionLoggerModule.scala │ │ │ ├── ProductMixerModule.scala │ │ │ ├── StratoClientModule.scala │ │ │ ├── product_mixer_flags/ │ │ │ │ ├── BUILD │ │ │ │ └── ProductMixerFlagModule.scala │ │ │ └── stringcenter/ │ │ │ ├── BUILD │ │ │ └── ProductScopeStringCenterModule.scala │ │ ├── pipeline/ │ │ │ ├── BUILD │ │ │ ├── CandidatePipelineFeatures.scala │ │ │ ├── FailOpenPolicy.scala │ │ │ ├── InvalidStepStateException.scala │ │ │ ├── NewPipelineArrowBuilder.scala │ │ │ ├── NewPipelineBuilder.scala │ │ │ ├── NewPipelineResult.scala │ │ │ ├── NewStepData.scala │ │ │ ├── Pipeline.scala │ │ │ ├── PipelineBuilder.scala │ │ │ ├── PipelineConfig.scala │ │ │ ├── PipelineCursor.scala │ │ │ ├── PipelineCursorSerializer.scala │ │ │ ├── PipelineQuery.scala │ │ │ ├── PipelineResult.scala │ │ │ ├── candidate/ │ │ │ │ ├── BUILD │ │ │ │ ├── CandidatePipeline.scala │ │ │ │ ├── CandidatePipelineBuilder.scala │ │ │ │ ├── CandidatePipelineBuilderFactory.scala │ │ │ │ ├── CandidatePipelineConfig.scala │ │ │ │ ├── CandidatePipelineResult.scala │ │ │ │ ├── PassthroughCandidatePipelineConfig.scala │ │ │ │ └── StaticCandidatePipelineConfig.scala │ │ │ ├── mixer/ │ │ │ │ ├── BUILD │ │ │ │ ├── MixerPipeline.scala │ │ │ │ ├── MixerPipelineBuilder.scala │ │ │ │ ├── MixerPipelineBuilderFactory.scala │ │ │ │ ├── MixerPipelineConfig.scala │ │ │ │ └── MixerPipelineResult.scala │ │ │ ├── pipeline_failure/ │ │ │ │ ├── BUILD │ │ │ │ ├── PipelineFailure.scala │ │ │ │ ├── PipelineFailureCategory.scala │ │ │ │ ├── PipelineFailureClassifier.scala │ │ │ │ └── PipelineFailureSerializer.scala │ │ │ ├── product/ │ │ │ │ ├── BUILD │ │ │ │ ├── ProductPipeline.scala │ │ │ │ ├── ProductPipelineBuilder.scala │ │ │ │ ├── ProductPipelineBuilderFactory.scala │ │ │ │ ├── ProductPipelineConfig.scala │ │ │ │ ├── ProductPipelineRequest.scala │ │ │ │ └── ProductPipelineResult.scala │ │ │ ├── recommendation/ │ │ │ │ ├── BUILD │ │ │ │ ├── RecommendationPipeline.scala │ │ │ │ ├── RecommendationPipelineBuilder.scala │ │ │ │ ├── RecommendationPipelineBuilderFactory.scala │ │ │ │ ├── RecommendationPipelineConfig.scala │ │ │ │ └── RecommendationPipelineResult.scala │ │ │ ├── scoring/ │ │ │ │ ├── BUILD │ │ │ │ ├── NewScoringPipelineBuilder.scala │ │ │ │ ├── ScoringPipeline.scala │ │ │ │ ├── ScoringPipelineBuilder.scala │ │ │ │ ├── ScoringPipelineBuilderFactory.scala │ │ │ │ ├── ScoringPipelineConfig.scala │ │ │ │ └── ScoringPipelineResult.scala │ │ │ ├── state/ │ │ │ │ ├── BUILD │ │ │ │ ├── HasAsyncFeatureMap.scala │ │ │ │ ├── HasCandidates.scala │ │ │ │ ├── HasCandidatesWithDetails.scala │ │ │ │ ├── HasCandidatesWithFeatures.scala │ │ │ │ ├── HasExecutorResults.scala │ │ │ │ ├── HasParams.scala │ │ │ │ ├── HasQuery.scala │ │ │ │ ├── HasRequest.scala │ │ │ │ └── HasResult.scala │ │ │ └── step/ │ │ │ ├── BUILD │ │ │ ├── Step.scala │ │ │ ├── async_feature_map/ │ │ │ │ ├── AsyncFeatureMapStep.scala │ │ │ │ └── BUILD.bazel │ │ │ ├── candidate_feature_hydrator/ │ │ │ │ ├── BUILD │ │ │ │ └── CandidateFeatureHydratorStep.scala │ │ │ ├── candidate_source/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── CandidateSourceStep.scala │ │ │ ├── decorator/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── DecoratorStep.scala │ │ │ ├── domain_marshaller/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── DomainMarshallerStep.scala │ │ │ ├── filter/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── FilterStep.scala │ │ │ ├── gate/ │ │ │ │ ├── BUILD │ │ │ │ └── GateStep.scala │ │ │ ├── group_results/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── GroupResultsStep.scala │ │ │ ├── pipeline_executor/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── PipelineExecutorStep.scala │ │ │ ├── pipeline_selector/ │ │ │ │ ├── BUILD │ │ │ │ └── PipelineSelectorStep.scala │ │ │ ├── quality_factor/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── QualityFactorStep.scala │ │ │ ├── query_feature_hydrator/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── QueryFeatureHydratorStep.scala │ │ │ ├── query_transformer/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── QueryTransformerStep.scala │ │ │ ├── scorer/ │ │ │ │ ├── BUILD │ │ │ │ └── ScorerStep.scala │ │ │ ├── selector/ │ │ │ │ ├── BUILD │ │ │ │ └── SelectorStep.scala │ │ │ ├── side_effect/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── SideEffectStep.scala │ │ │ └── transport_marshaller/ │ │ │ ├── BUILD.bazel │ │ │ └── TransportMarshallerStep.scala │ │ ├── product/ │ │ │ ├── BUILD │ │ │ ├── ProductParamConfig.scala │ │ │ ├── ProductParamConfigBuilder.scala │ │ │ ├── guice/ │ │ │ │ ├── BUILD │ │ │ │ ├── ProductScope.scala │ │ │ │ ├── ProductScopeModule.scala │ │ │ │ └── SimpleScope.scala │ │ │ └── registry/ │ │ │ ├── BUILD │ │ │ ├── ProductParamRegistry.scala │ │ │ ├── ProductPipelineRegistry.scala │ │ │ └── ProductPipelineRegistryConfig.scala │ │ ├── quality_factor/ │ │ │ ├── BUILD │ │ │ ├── Bounds.scala │ │ │ ├── LinearLatencyQualityFactor.scala │ │ │ ├── LinearLatencyQualityFactorObserver.scala │ │ │ ├── QualityFactor.scala │ │ │ ├── QualityFactorConfig.scala │ │ │ ├── QualityFactorObserver.scala │ │ │ ├── QualityFactorStatus.scala │ │ │ ├── QueriesPerSecondBasedQualityFactor.scala │ │ │ ├── QueriesPerSecondBasedQualityFactorObserver.scala │ │ │ └── QueryRateCounter.scala │ │ ├── service/ │ │ │ ├── BUILD │ │ │ ├── Executor.scala │ │ │ ├── ExecutorObserver.scala │ │ │ ├── ExecutorResult.scala │ │ │ ├── async_feature_map_executor/ │ │ │ │ ├── AsyncFeatureMapExecutor.scala │ │ │ │ └── BUILD │ │ │ ├── candidate_decorator_executor/ │ │ │ │ ├── BUILD │ │ │ │ ├── CandidateDecoratorExecutor.scala │ │ │ │ └── CandidateDecoratorExecutorResult.scala │ │ │ ├── candidate_feature_hydrator_executor/ │ │ │ │ ├── BUILD │ │ │ │ ├── CandidateFeatureHydratorExecutor.scala │ │ │ │ └── CandidateFeatureHydratorExecutorResult.scala │ │ │ ├── candidate_feature_transformer_executor/ │ │ │ │ ├── BUILD │ │ │ │ ├── CandidateFeatureTransformerExecutor.scala │ │ │ │ └── CandidateFeatureTransformerExecutorResult.scala │ │ │ ├── candidate_pipeline_executor/ │ │ │ │ ├── BUILD │ │ │ │ ├── CandidatePipelineExecutor.scala │ │ │ │ └── CandidatePipelineExecutorResult.scala │ │ │ ├── candidate_source_executor/ │ │ │ │ ├── BUILD │ │ │ │ ├── CandidateSourceExecutor.scala │ │ │ │ └── CandidateSourceExecutorResult.scala │ │ │ ├── component_registry/ │ │ │ │ ├── BUILD │ │ │ │ ├── ComponentRegistry.scala │ │ │ │ └── RegisteredComponent.scala │ │ │ ├── debug_query/ │ │ │ │ ├── AuthorizationService.scala │ │ │ │ ├── BUILD │ │ │ │ ├── DebugQueryNotSupportedService.scala │ │ │ │ ├── DebugQueryService.scala │ │ │ │ └── ParamsSerializerModule.scala │ │ │ ├── domain_marshaller_executor/ │ │ │ │ ├── BUILD │ │ │ │ └── DomainMarshallerExecutor.scala │ │ │ ├── feature_hydrator_observer/ │ │ │ │ ├── BUILD │ │ │ │ └── FeatureHydratorObserver.scala │ │ │ ├── filter_executor/ │ │ │ │ ├── BUILD │ │ │ │ ├── FilterExecutor.scala │ │ │ │ └── FilterExecutorResult.scala │ │ │ ├── gate_executor/ │ │ │ │ ├── BUILD │ │ │ │ ├── ExecutedGateResult.scala │ │ │ │ ├── GateExecutor.scala │ │ │ │ ├── GateExecutorResult.scala │ │ │ │ └── StoppedGateException.scala │ │ │ ├── group_results_executor/ │ │ │ │ ├── BUILD │ │ │ │ └── GroupResultsExecutor.scala │ │ │ ├── pipeline_execution_logger/ │ │ │ │ ├── AllowListedPipelineExecutionLogger.scala │ │ │ │ ├── BUILD │ │ │ │ └── PipelineExecutionLogger.scala │ │ │ ├── pipeline_executor/ │ │ │ │ ├── BUILD │ │ │ │ ├── PipelineExecutor.scala │ │ │ │ └── PipelineExecutorResult.scala │ │ │ ├── pipeline_result_side_effect_executor/ │ │ │ │ ├── BUILD │ │ │ │ └── PipelineResultSideEffectExecutor.scala │ │ │ ├── pipeline_selector_executor/ │ │ │ │ ├── BUILD │ │ │ │ ├── PipelineSelectorExecutor.scala │ │ │ │ └── PipelineSelectorExecutorResult.scala │ │ │ ├── quality_factor_executor/ │ │ │ │ ├── BUILD │ │ │ │ └── QualityFactorExecutorResult.scala │ │ │ ├── query_feature_hydrator_executor/ │ │ │ │ ├── AsyncIndividualFeatureHydratorResultSerializer.scala │ │ │ │ ├── BUILD │ │ │ │ └── QueryFeatureHydratorExecutor.scala │ │ │ ├── scoring_pipeline_executor/ │ │ │ │ ├── BUILD │ │ │ │ ├── ScoringPipelineExecutor.scala │ │ │ │ └── ScoringPipelineExecutorResult.scala │ │ │ ├── selector_executor/ │ │ │ │ ├── BUILD │ │ │ │ ├── SelectorExecutor.scala │ │ │ │ └── SelectorExecutorResult.scala │ │ │ ├── slice/ │ │ │ │ ├── BUILD │ │ │ │ └── SliceService.scala │ │ │ ├── transformer_executor/ │ │ │ │ ├── BUILD │ │ │ │ ├── PerCandidateTransformerExecutor.scala │ │ │ │ └── TransformerExecutor.scala │ │ │ ├── transport_marshaller_executor/ │ │ │ │ ├── BUILD │ │ │ │ └── TransportMarshallerExecutor.scala │ │ │ ├── urp/ │ │ │ │ ├── BUILD │ │ │ │ └── UrpService.scala │ │ │ └── urt/ │ │ │ ├── BUILD │ │ │ └── UrtService.scala │ │ └── util/ │ │ ├── BUILD │ │ ├── FuturePools.scala │ │ ├── OffloadFuturePools.scala │ │ └── SortIndexBuilder.scala │ └── shared-library/ │ └── src/ │ └── main/ │ └── scala/ │ └── com/ │ └── twitter/ │ └── product_mixer/ │ └── shared_library/ │ ├── http_client/ │ │ ├── BUILD │ │ ├── FinagleHttpClientBuilder.scala │ │ ├── FinagleHttpClientWithProxyBuilder.scala │ │ └── HttpHostPort.scala │ ├── manhattan_client/ │ │ ├── BUILD │ │ └── ManhattanClientBuilder.scala │ ├── memcached_client/ │ │ ├── BUILD │ │ └── MemcachedClientBuilder.scala │ ├── observer/ │ │ ├── BUILD │ │ ├── Observer.scala │ │ ├── ResultsObserver.scala │ │ └── ResultsStatsObserver.scala │ └── thrift_client/ │ ├── BUILD │ └── FinagleThriftClientBuilder.scala ├── pushservice/ │ ├── BUILD.bazel │ ├── README.md │ └── src/ │ └── main/ │ ├── python/ │ │ └── models/ │ │ ├── heavy_ranking/ │ │ │ ├── BUILD │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── deep_norm.py │ │ │ ├── eval.py │ │ │ ├── features.py │ │ │ ├── graph.py │ │ │ ├── lib/ │ │ │ │ ├── BUILD │ │ │ │ ├── layers.py │ │ │ │ ├── model.py │ │ │ │ └── params.py │ │ │ ├── model_pools.py │ │ │ ├── params.py │ │ │ ├── run_args.py │ │ │ └── update_warm_start_checkpoint.py │ │ ├── libs/ │ │ │ ├── BUILD │ │ │ ├── __init__.py │ │ │ ├── customized_full_sparse.py │ │ │ ├── get_feat_config.py │ │ │ ├── graph_utils.py │ │ │ ├── group_metrics.py │ │ │ ├── initializer.py │ │ │ ├── light_ranking_metrics.py │ │ │ ├── metric_fn_utils.py │ │ │ ├── model_args.py │ │ │ ├── model_utils.py │ │ │ └── warm_start_utils.py │ │ └── light_ranking/ │ │ ├── BUILD │ │ ├── README.md │ │ ├── __init__.py │ │ ├── deep_norm.py │ │ ├── eval_model.py │ │ └── model_pools_mlp.py │ └── scala/ │ └── com/ │ └── twitter/ │ └── frigate/ │ └── pushservice/ │ ├── BUILD.bazel │ ├── PushMixerThriftServerWarmupHandler.scala │ ├── PushServiceMain.scala │ ├── adaptor/ │ │ ├── ContentRecommenderMixerAdaptor.scala │ │ ├── EarlyBirdFirstDegreeCandidateAdaptor.scala │ │ ├── ExploreVideoTweetCandidateAdaptor.scala │ │ ├── FRSTweetCandidateAdaptor.scala │ │ ├── GenericCandidateAdaptor.scala │ │ ├── HighQualityTweetsAdaptor.scala │ │ ├── ListsToRecommendCandidateAdaptor.scala │ │ ├── LoggedOutPushCandidateSourceGenerator.scala │ │ ├── OnboardingPushCandidateAdaptor.scala │ │ ├── PushCandidateSourceGenerator.scala │ │ ├── TopTweetImpressionsCandidateAdaptor.scala │ │ ├── TopTweetsByGeoAdaptor.scala │ │ ├── TrendsCandidatesAdaptor.scala │ │ └── TripGeoCandidatesAdaptor.scala │ ├── config/ │ │ ├── Config.scala │ │ ├── DeployConfig.scala │ │ ├── ExperimentsWithStats.scala │ │ ├── ProdConfig.scala │ │ ├── StagingConfig.scala │ │ └── mlconfig/ │ │ └── DeepbirdV2ModelConfig.scala │ ├── controller/ │ │ └── PushServiceController.scala │ ├── exception/ │ │ ├── DisplayLocationNotSupportedException.scala │ │ ├── InvalidSportDomainException.scala │ │ ├── TweetNTabRequestHydratorException.scala │ │ ├── UnsupportedCrtException.scala │ │ └── UttEntityNotFoundException.scala │ ├── ml/ │ │ ├── HealthFeatureGetter.scala │ │ ├── HydrationContextBuilder.scala │ │ └── PushMLModelScorer.scala │ ├── model/ │ │ ├── DiscoverTwitter.scala │ │ ├── F1FirstdegreeTweet.scala │ │ ├── ListRecommendationPushCandidate.scala │ │ ├── MagicFanoutCreatorEventPushCandidate.scala │ │ ├── MagicFanoutEventPushCandidate.scala │ │ ├── MagicFanoutHydratedCandidate.scala │ │ ├── MagicFanoutNewsEvent.scala │ │ ├── MagicFanoutProductLaunchPushCandidate.scala │ │ ├── MagicFanoutSportsPushCandidate.scala │ │ ├── OutOfNetworkTweetPushCandidate.scala │ │ ├── PushTypes.scala │ │ ├── ScheduledSpaceSpeaker.scala │ │ ├── ScheduledSpaceSubscriber.scala │ │ ├── SubscribedSearchTweetPushCandidate.scala │ │ ├── TopTweetImpressionsPushCandidate.scala │ │ ├── TopicProofTweetPushCandidate.scala │ │ ├── TrendTweetPushCandidate.scala │ │ ├── TripTweetPushCandidate.scala │ │ ├── TweetAction.scala │ │ ├── TweetFavorite.scala │ │ ├── TweetRetweet.scala │ │ ├── candidate/ │ │ │ ├── CopyInfo.scala │ │ │ ├── MLScores.scala │ │ │ ├── QualityScribing.scala │ │ │ └── Scriber.scala │ │ ├── ibis/ │ │ │ ├── CustomConfigurationMapForIbis.scala │ │ │ ├── DiscoverTwitterPushIbis2Hydrator.scala │ │ │ ├── F1FirstDegreeTweetIbis2Hydrator.scala │ │ │ ├── Ibis2Hydrator.scala │ │ │ ├── InlineActionIbis2Hydrator.scala │ │ │ ├── ListIbis2Hydrator.scala │ │ │ ├── MagicFanoutCreatorEventIbis2Hydrator.scala │ │ │ ├── MagicFanoutNewsEventIbis2Hydrator.scala │ │ │ ├── MagicFanoutProductLaunchIbis2Hydrator.scala │ │ │ ├── MagicFanoutSportsEventIbis2Hydrator.scala │ │ │ ├── OutOfNetworkTweetIbis2Hydrator.scala │ │ │ ├── OverrideForIbis2Request.scala │ │ │ ├── PushOverrideInfo.scala │ │ │ ├── RankedSocialContextIbis2Hydrator.scala │ │ │ ├── ScheduledSpaceSpeakerIbis2Hydrator.scala │ │ │ ├── ScheduledSpaceSubscriberIbis2Hydrator.scala │ │ │ ├── SubscribedSearchTweetIbis2Hydrator.scala │ │ │ ├── TopTweetImpressionsCandidateIbis2Hydrator.scala │ │ │ ├── TopicProofTweetIbis2Hydrator.scala │ │ │ ├── TrendTweetIbis2Hydrator.scala │ │ │ ├── TweetCandidateIbis2Hydrator.scala │ │ │ ├── TweetFavoriteIbis2Hydrator.scala │ │ │ └── TweetRetweetIbis2Hydrator.scala │ │ └── ntab/ │ │ ├── CandidateNTabCopy.scala │ │ ├── DiscoverTwitterNtabRequestHydrator.scala │ │ ├── EventNTabRequestHydrator.scala │ │ ├── F1FirstDegreeTweetNTabRequestHydrator.scala │ │ ├── ListCandidateNTabRequestHydrator.scala │ │ ├── MagicFanoutCreatorEventNtabRequestHydrator.scala │ │ ├── MagicFanoutNewsEventNTabRequestHydrator.scala │ │ ├── MagicFanoutProductLaunchNtabRequestHydrator.scala │ │ ├── MagicFanoutSportsEventNTabRequestHydrator.scala │ │ ├── NTabRequest.scala │ │ ├── NTabRequestHydrator.scala │ │ ├── NTabSocialContext.scala │ │ ├── OutOfNetworkTweetNTabRequestHydrator.scala │ │ ├── ScheduledSpaceNTabRequestHydrator.scala │ │ ├── SubscribedSearchTweetNtabRequestHydrator.scala │ │ ├── TopTweetImpressionsNTabRequestHydrator.scala │ │ ├── TopicProofTweetNtabRequestHydrator.scala │ │ ├── TrendTweetNtabHydrator.scala │ │ ├── TweetFavoriteNTabRequestHydrator.scala │ │ ├── TweetNTabRequestHydrator.scala │ │ └── TweetRetweetNTabRequestHydrator.scala │ ├── module/ │ │ ├── DeployConfigModule.scala │ │ ├── FilterModule.scala │ │ ├── FlagModule.scala │ │ ├── LoggedOutPushTargetUserBuilderModule.scala │ │ ├── PushHandlerModule.scala │ │ ├── PushServiceDarkTrafficModule.scala │ │ ├── PushTargetUserBuilderModule.scala │ │ └── ThriftWebFormsModule.scala │ ├── params/ │ │ ├── DeciderKey.scala │ │ ├── PushConstants.scala │ │ ├── PushEnums.scala │ │ ├── PushFeatureSwitchParams.scala │ │ ├── PushFeatureSwitches.scala │ │ ├── PushMLModelParams.scala │ │ ├── PushParams.scala │ │ ├── PushServiceTunableKeys.scala │ │ └── ShardParams.scala │ ├── predicate/ │ │ ├── BigFilteringEpsilonGreedyExplorationPredicate.scala │ │ ├── BqmlHealthModelPredicates.scala │ │ ├── BqmlQualityModelPredicates.scala │ │ ├── CaretFeedbackHistoryFilter.scala │ │ ├── CasLockPredicate.scala │ │ ├── CrtDeciderPredicate.scala │ │ ├── DiscoverTwitterPredicate.scala │ │ ├── FatiguePredicate.scala │ │ ├── HealthPredicates.scala │ │ ├── JointDauAndQualityModelPredicate.scala │ │ ├── ListPredicates.scala │ │ ├── LoggedOutPreRankingPredicates.scala │ │ ├── LoggedOutTargetPredicates.scala │ │ ├── MlModelsHoldbackExperimentPredicate.scala │ │ ├── OONSpreadControlPredicate.scala │ │ ├── OONTweetNegativeFeedbackBasedPredicate.scala │ │ ├── OutOfNetworkCandidatesQualityPredicates.scala │ │ ├── PNegMultimodalPredicates.scala │ │ ├── PostRankingPredicateHelper.scala │ │ ├── PreRankingPredicates.scala │ │ ├── PredicatesForCandidate.scala │ │ ├── SGSPredicatesForCandidate.scala │ │ ├── ScarecrowPredicate.scala │ │ ├── SpacePredicate.scala │ │ ├── TargetEngagementPredicate.scala │ │ ├── TargetNtabCaretClickFatiguePredicate.scala │ │ ├── TargetPredicates.scala │ │ ├── TopTweetImpressionsPredicates.scala │ │ ├── TweetEngagementRatioPredicate.scala │ │ ├── TweetLanguagePredicate.scala │ │ ├── TweetWithheldContentPredicate.scala │ │ ├── event/ │ │ │ └── EventPredicatesForCandidate.scala │ │ ├── magic_fanout/ │ │ │ ├── MagicFanoutPredicatesForCandidate.scala │ │ │ ├── MagicFanoutPredicatesUtil.scala │ │ │ ├── MagicFanoutSportsUtil.scala │ │ │ └── MagicFanoutTargetingPredicateWrappersForCandidate.scala │ │ ├── ntab_caret_fatigue/ │ │ │ ├── CRTBasedNtabCaretClickFatiguePredicates.scala │ │ │ ├── ContinuousFunction.scala │ │ │ ├── FeedbackModel.scala │ │ │ ├── MagicFanoutNtabCaretFatiguePredicate.scala │ │ │ ├── NtabCaretClickCandidateFatiguePredicate.scala │ │ │ ├── NtabCaretClickFatiguePredicate.scala │ │ │ ├── NtabCaretClickFatigueUtils.scala │ │ │ └── RecTypeNtabCaretFatiguePredicate.scala │ │ ├── package.scala │ │ └── quality_model_predicate/ │ │ ├── OpenOrNtabClickQualityPredicate.scala │ │ ├── QualityPredicateCommon.scala │ │ └── QualityPredicateMap.scala │ ├── rank/ │ │ ├── CRTBoostRanker.scala │ │ ├── CRTDownRanker.scala │ │ ├── LoggedOutRanker.scala │ │ ├── ModelBasedRanker.scala │ │ ├── PushserviceRanker.scala │ │ ├── RFPHLightRanker.scala │ │ ├── RFPHRanker.scala │ │ └── SubscriptionCreatorRanker.scala │ ├── refresh_handler/ │ │ ├── LoggedOutRefreshForPushHandler.scala │ │ ├── PushCandidateHydrator.scala │ │ ├── RFPHFeatureHydrator.scala │ │ ├── RFPHPrerankFilter.scala │ │ ├── RFPHRestrictStep.scala │ │ ├── RFPHStatsRecorder.scala │ │ ├── RefreshForPushHandler.scala │ │ ├── RefreshForPushNotifier.scala │ │ └── cross/ │ │ ├── BaseCopyFramework.scala │ │ ├── CandidateCopyExpansion.scala │ │ ├── CandidateCopyPair.scala │ │ ├── CandidateToCopy.scala │ │ ├── CopyFilters.scala │ │ └── CopyPredicates.scala │ ├── scriber/ │ │ └── MrRequestScribeHandler.scala │ ├── send_handler/ │ │ ├── SendHandler.scala │ │ ├── SendHandlerPushCandidateHydrator.scala │ │ └── generator/ │ │ ├── CandidateGenerator.scala │ │ ├── MagicFanoutCreatorEventCandidateGenerator.scala │ │ ├── MagicFanoutNewsEventCandidateGenerator.scala │ │ ├── MagicFanoutProductLaunchCandidateGenerator.scala │ │ ├── MagicFanoutSportsEventCandidateGenerator.scala │ │ ├── PushRequestToCandidate.scala │ │ ├── ScheduledSpaceSpeakerCandidateGenerator.scala │ │ └── ScheduledSpaceSubscriberCandidateGenerator.scala │ ├── store/ │ │ ├── ContentMixerStore.scala │ │ ├── CopySelectionServiceStore.scala │ │ ├── CrMixerTweetStore.scala │ │ ├── ExploreRankerStore.scala │ │ ├── FollowRecommendationsStore.scala │ │ ├── IbisStore.scala │ │ ├── InterestDiscoveryStore.scala │ │ ├── LabeledPushRecsDecideredStore.scala │ │ ├── LexServiceStore.scala │ │ ├── NTabHistoryStore.scala │ │ ├── OCFPromptHistoryStore.scala │ │ ├── OnlineUserHistoryStore.scala │ │ ├── OpenAppUserStore.scala │ │ ├── SocialGraphServiceProcessStore.scala │ │ ├── SoftUserFollowingStore.scala │ │ ├── TweetImpressionsStore.scala │ │ ├── TweetTranslationStore.scala │ │ └── UttEntityHydrationStore.scala │ ├── take/ │ │ ├── CandidateNotifier.scala │ │ ├── LoggedOutRefreshForPushNotifier.scala │ │ ├── NotificationSender.scala │ │ ├── NotificationServiceSender.scala │ │ ├── SendHandlerNotifier.scala │ │ ├── candidate_validator/ │ │ │ ├── CandidateValidator.scala │ │ │ ├── RFPHCandidateValidator.scala │ │ │ ├── SendHandlerPostCandidateValidator.scala │ │ │ └── SendHandlerPreCandidateValidator.scala │ │ ├── channel_selection/ │ │ │ ├── ChannelCandidate.scala │ │ │ ├── ChannelSelector.scala │ │ │ └── NtabOnlyChannelSelector.scala │ │ ├── history/ │ │ │ ├── EventBusWriter.scala │ │ │ └── HistoryWriter.scala │ │ ├── predicates/ │ │ │ ├── BasicRFPHPredicates.scala │ │ │ ├── BasicSendHandlerPredicates.scala │ │ │ ├── BasicTweetPredicates.scala │ │ │ ├── BasicTweetPredicatesForRFPH.scala │ │ │ ├── OutOfNetworkTweetPredicates.scala │ │ │ ├── TakeCommonPredicates.scala │ │ │ └── candidate_map/ │ │ │ ├── CandidatePredicatesMap.scala │ │ │ └── SendHandlerCandidatePredicatesMap.scala │ │ └── sender/ │ │ ├── Ibis2Sender.scala │ │ └── NtabSender.scala │ ├── target/ │ │ ├── CustomFSFields.scala │ │ ├── LoggedOutPushTargetUserBuilder.scala │ │ ├── PushTargetUserBuilder.scala │ │ ├── RFPHTargetPredicateGenerator.scala │ │ ├── TargetAppPermissions.scala │ │ └── TargetScoringDetails.scala │ └── util/ │ ├── AdaptorUtils.scala │ ├── AdhocStatsUtil.scala │ ├── Candidate2FrigateNotification.scala │ ├── CandidateHydrationUtil.scala │ ├── CandidateUtil.scala │ ├── CopyUtil.scala │ ├── EmailLandingPageExperimentUtil.scala │ ├── FunctionalUtil.scala │ ├── IbisScribeTargets.scala │ ├── InlineActionUtil.scala │ ├── MediaAnnotationsUtil.scala │ ├── MinDurationModifierCalculator.scala │ ├── MrUserStateUtil.scala │ ├── NsfwPersonalizationUtil.scala │ ├── OverrideNotificationUtil.scala │ ├── PushAdaptorUtil.scala │ ├── PushAppPermissionUtil.scala │ ├── PushCapUtil.scala │ ├── PushDeviceUtil.scala │ ├── PushIbisUtil.scala │ ├── PushToHomeUtil.scala │ ├── RFPHTakeStepUtil.scala │ ├── RelationshipUtil.scala │ ├── ResponseStatsTrackUtils.scala │ ├── SendHandlerPredicateUtil.scala │ └── TopicsUtil.scala ├── recos-injector/ │ ├── BUILD.bazel │ ├── CONFIG.ini │ ├── README.md │ └── server/ │ ├── BUILD │ ├── config/ │ │ ├── BUILD │ │ ├── change_log_config.ini │ │ └── decider.yml │ └── src/ │ └── main/ │ └── scala/ │ └── com/ │ └── twitter/ │ └── recosinjector/ │ ├── BUILD │ ├── Main.scala │ ├── clients/ │ │ ├── BUILD │ │ ├── Gizmoduck.scala │ │ ├── RecosHoseEntitiesCache.scala │ │ ├── SocialGraph.scala │ │ ├── Tweetypie.scala │ │ └── UrlResolver.scala │ ├── config/ │ │ ├── BUILD │ │ ├── CacheConfig.scala │ │ ├── Config.scala │ │ ├── DeployConfig.scala │ │ ├── ProdConfig.scala │ │ └── StagingConfig.scala │ ├── decider/ │ │ ├── BUILD │ │ └── RecosInjectorDecider.scala │ ├── edges/ │ │ ├── BUILD │ │ ├── Edges.scala │ │ ├── EventToMessageBuilder.scala │ │ ├── SocialWriteEventToUserUserGraphBuilder.scala │ │ ├── TimelineEventToUserTweetEntityGraphBuilder.scala │ │ ├── TimelineEventToUserTweetGraphBuilder.scala │ │ ├── TweetEventToUserTweetEntityGraphBuilder.scala │ │ ├── TweetEventToUserTweetGraphBuilder.scala │ │ ├── TweetEventToUserUserGraphBuilder.scala │ │ ├── UnifiedUserActionToUserAdGraphBuilder.scala │ │ ├── UnifiedUserActionToUserTweetGraphPlusBuilder.scala │ │ ├── UnifiedUserActionToUserVideoGraphBuilder.scala │ │ └── UserTweetEntityEdgeBuilder.scala │ ├── event_processors/ │ │ ├── BUILD │ │ ├── EventBusProcessor.scala │ │ ├── SocialWriteEventProcessor.scala │ │ ├── TimelineEventProcessor.scala │ │ └── TweetEventProcessor.scala │ ├── filters/ │ │ ├── BUILD │ │ ├── NullCastTweetFilter.scala │ │ ├── TweetFilter.scala │ │ └── UserFilter.scala │ ├── publishers/ │ │ ├── BUILD │ │ └── KafkaEventPublisher.scala │ ├── util/ │ │ ├── BUILD │ │ └── EventDetails.scala │ └── uua_processors/ │ ├── BUILD │ ├── UnifiedUserActionProcessor.scala │ └── UnifiedUserActionsConsumer.scala ├── representation-manager/ │ ├── BUILD.bazel │ ├── README.md │ ├── bin/ │ │ └── deploy.sh │ ├── client/ │ │ └── src/ │ │ └── main/ │ │ └── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── representation_manager/ │ │ ├── BUILD │ │ ├── StoreBuilder.scala │ │ └── config/ │ │ ├── BUILD │ │ ├── ClientConfig.scala │ │ └── InMemoryCacheConfig.scala │ └── server/ │ ├── BUILD │ └── src/ │ └── main/ │ ├── resources/ │ │ ├── BUILD │ │ ├── config/ │ │ │ └── decider.yml │ │ └── logback.xml │ ├── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── representation_manager/ │ │ ├── BUILD │ │ ├── RepresentationManagerFedServer.scala │ │ ├── columns/ │ │ │ ├── BUILD │ │ │ ├── ColumnConfigBase.scala │ │ │ ├── topic/ │ │ │ │ ├── BUILD │ │ │ │ ├── LocaleEntityIdSimClustersEmbeddingCol.scala │ │ │ │ └── TopicIdSimClustersEmbeddingCol.scala │ │ │ ├── tweet/ │ │ │ │ ├── BUILD │ │ │ │ └── TweetSimClustersEmbeddingCol.scala │ │ │ └── user/ │ │ │ ├── BUILD │ │ │ └── UserSimClustersEmbeddingCol.scala │ │ ├── common/ │ │ │ ├── BUILD │ │ │ ├── MemCacheConfig.scala │ │ │ └── RepresentationManagerDecider.scala │ │ ├── migration/ │ │ │ ├── BUILD │ │ │ └── LegacyRMS.scala │ │ ├── modules/ │ │ │ ├── BUILD │ │ │ ├── CacheModule.scala │ │ │ ├── InterestsThriftClientModule.scala │ │ │ ├── LegacyRMSConfigModule.scala │ │ │ ├── StoreModule.scala │ │ │ ├── TimerModule.scala │ │ │ └── UttClientModule.scala │ │ └── store/ │ │ ├── BUILD │ │ ├── DeciderConstants.scala │ │ ├── TopicSimClustersEmbeddingStore.scala │ │ ├── TweetSimClustersEmbeddingStore.scala │ │ └── UserSimClustersEmbeddingStore.scala │ └── thrift/ │ ├── BUILD │ └── com/ │ └── twitter/ │ └── representation_manager/ │ └── service.thrift ├── representation-scorer/ │ ├── BUILD.bazel │ ├── README.md │ ├── bin/ │ │ ├── canary-check.sh │ │ ├── deploy.sh │ │ └── remote-debug-tunnel.sh │ ├── docs/ │ │ └── index.rst │ └── server/ │ ├── BUILD │ └── src/ │ └── main/ │ ├── resources/ │ │ ├── BUILD │ │ ├── com/ │ │ │ └── twitter/ │ │ │ └── slo/ │ │ │ └── slo.json │ │ ├── config/ │ │ │ └── decider.yml │ │ └── logback.xml │ ├── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── representationscorer/ │ │ ├── BUILD │ │ ├── RepresentationScorerFedServer.scala │ │ ├── columns/ │ │ │ ├── BUILD │ │ │ ├── Info.scala │ │ │ ├── ListScoreColumn.scala │ │ │ ├── ScoreColumn.scala │ │ │ ├── SimClustersRecentEngagementSimilarityColumn.scala │ │ │ └── SimClustersRecentEngagementSimilarityUserTweetEdgeColumn.scala │ │ ├── common/ │ │ │ ├── BUILD │ │ │ ├── DeciderConstants.scala │ │ │ ├── RepresentationScorerDecider.scala │ │ │ └── package.scala │ │ ├── modules/ │ │ │ ├── BUILD │ │ │ ├── CacheModule.scala │ │ │ ├── EmbeddingStoreModule.scala │ │ │ ├── RMSConfigModule.scala │ │ │ └── TimerModule.scala │ │ ├── scorestore/ │ │ │ ├── BUILD │ │ │ ├── ScoreStore.scala │ │ │ ├── TopicTweetCertoScoreStore.scala │ │ │ ├── TopicTweetRankingScoreStore.scala │ │ │ └── TopicTweetsCosineSimilarityAggregateStore.scala │ │ └── twistlyfeatures/ │ │ ├── BUILD │ │ ├── Engagements.scala │ │ ├── ScoreResult.scala │ │ ├── Scorer.scala │ │ ├── UserSignalServiceRecentEngagementsClient.scala │ │ └── UserSignalServiceRecentEngagementsClientModule.scala │ └── thrift/ │ ├── BUILD │ └── com/ │ └── twitter/ │ └── representationscorer/ │ └── service.thrift ├── science/ │ └── search/ │ └── ingester/ │ └── config/ │ ├── README.md │ ├── pipeline-indexer.userupdates.xml │ ├── pipeline-ingester.protected.xml │ ├── pipeline-ingester.realtime.xml │ └── pipeline-ingester.realtime_cg.xml ├── simclusters-ann/ │ ├── BUILD.bazel │ ├── README.md │ ├── server/ │ │ ├── BUILD │ │ └── src/ │ │ └── main/ │ │ ├── resources/ │ │ │ ├── BUILD │ │ │ ├── config/ │ │ │ │ └── decider.yml │ │ │ └── logback.xml │ │ └── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── simclustersann/ │ │ ├── BUILD │ │ ├── SimclustersAnnServer.scala │ │ ├── SimclustersAnnWarmupHandler.scala │ │ ├── candidate_source/ │ │ │ ├── ApproximateCosineSimilarity.scala │ │ │ ├── BUILD │ │ │ ├── ExperimentalApproximateCosineSimilarity.scala │ │ │ ├── OptimizedApproximateCosineSimilarity.scala │ │ │ └── SimClustersANNCandidateSource.scala │ │ ├── common/ │ │ │ ├── BUILD │ │ │ └── FlagNames.scala │ │ ├── controllers/ │ │ │ ├── BUILD │ │ │ └── SimClustersANNController.scala │ │ ├── exceptions/ │ │ │ ├── BUILD │ │ │ ├── InvalidRequestForSimClustersAnnVariantException.scala │ │ │ ├── InvalidRequestForSimClustersAnnVariantExceptionMapper.scala │ │ │ └── MissingClusterConfigForSimClustersAnnVariantException.scala │ │ ├── filters/ │ │ │ ├── BUILD │ │ │ ├── GetTweetCandidatesResponseStatsFilter.scala │ │ │ └── SimClustersAnnVariantFilter.scala │ │ └── modules/ │ │ ├── BUILD │ │ ├── CacheModule.scala │ │ ├── ClusterConfigMapperModule.scala │ │ ├── ClusterConfigModule.scala │ │ ├── ClusterTweetIndexProviderModule.scala │ │ ├── CustomMtlsThriftWebFormsModule.scala │ │ ├── EmbeddingStoreModule.scala │ │ ├── FlagsModule.scala │ │ ├── FuturePoolProvider.scala │ │ ├── RateLimiterModule.scala │ │ ├── ServiceNameMapperModule.scala │ │ ├── SimClustersANNCandidateSourceModule.scala │ │ └── StratoClientProviderModule.scala │ └── thrift/ │ └── src/ │ └── main/ │ └── thrift/ │ ├── BUILD │ └── simClustersAnn.thrift ├── src/ │ ├── java/ │ │ └── com/ │ │ └── twitter/ │ │ └── search/ │ │ ├── README.md │ │ ├── common/ │ │ │ ├── README.md │ │ │ ├── converter/ │ │ │ │ └── earlybird/ │ │ │ │ ├── BUILD │ │ │ │ ├── BasicIndexingConverter.java │ │ │ │ ├── CombinedIndexingConverter.java │ │ │ │ ├── DelayedIndexingConverter.java │ │ │ │ └── EncodedFeatureBuilder.java │ │ │ ├── encoding/ │ │ │ │ ├── docvalues/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── CSFTypeUtil.java │ │ │ │ └── features/ │ │ │ │ ├── BUILD │ │ │ │ ├── BinByteNormalizer.java │ │ │ │ ├── ByteNormalizer.java │ │ │ │ ├── ClampByteNormalizer.java │ │ │ │ ├── EncodedFeatures.java │ │ │ │ ├── IntNormalizer.java │ │ │ │ ├── IntegerEncodedFeatures.java │ │ │ │ ├── LogByteNormalizer.java │ │ │ │ ├── PredictionScoreNormalizer.java │ │ │ │ ├── SingleBytePositiveFloatNormalizer.java │ │ │ │ ├── SingleBytePositiveFloatUtil.java │ │ │ │ └── SmartIntegerNormalizer.java │ │ │ ├── query/ │ │ │ │ ├── BUILD │ │ │ │ ├── BoostUtils.java │ │ │ │ ├── CollectAnnotationsVisitor.java │ │ │ │ ├── CollectQueryTypeVisitor.java │ │ │ │ ├── CollectVariantVisitor.java │ │ │ │ ├── DefaultFilterWeight.java │ │ │ │ ├── DocIdFilter.java │ │ │ │ ├── FieldRankHitInfo.java │ │ │ │ ├── FieldWeightUtil.java │ │ │ │ ├── FilteredQuery.java │ │ │ │ ├── FilteredScorer.java │ │ │ │ ├── HitAttributeCollector.java │ │ │ │ ├── HitAttributeHelper.java │ │ │ │ ├── HitAttributeProvider.java │ │ │ │ ├── IDDisjunctionQuery.java │ │ │ │ ├── IdentifiableQuery.java │ │ │ │ ├── IdentifiableQueryScorer.java │ │ │ │ ├── IdentifiableQueryWeight.java │ │ │ │ ├── MappableField.java │ │ │ │ ├── MultiTermDisjunctionQuery.java │ │ │ │ ├── QueryCommonFieldHitsVisitor.java │ │ │ │ ├── QueryHitAttributeHelper.java │ │ │ │ ├── QueryRankVisitor.java │ │ │ │ ├── SingleDocDocIdSetIterator.java │ │ │ │ └── StaticHitAttributeProvider.java │ │ │ ├── relevance/ │ │ │ │ ├── BUILD │ │ │ │ ├── NGramCache.java │ │ │ │ ├── TrendsThriftDataServiceManager.java │ │ │ │ ├── classifiers/ │ │ │ │ │ ├── TweetClassifier.java │ │ │ │ │ ├── TweetEvaluator.java │ │ │ │ │ ├── TweetOffensiveEvaluator.java │ │ │ │ │ ├── TweetQualityFeatureExtractor.java │ │ │ │ │ ├── TweetTextClassifier.java │ │ │ │ │ ├── TweetTextEvaluator.java │ │ │ │ │ └── TweetTrendsExtractor.java │ │ │ │ ├── config/ │ │ │ │ │ └── TweetProcessingConfig.java │ │ │ │ ├── entities/ │ │ │ │ │ ├── GeoObject.java │ │ │ │ │ ├── PotentialLocationObject.java │ │ │ │ │ ├── TwitterMessage.java │ │ │ │ │ ├── TwitterMessageUser.java │ │ │ │ │ ├── TwitterMessageUtil.java │ │ │ │ │ ├── TwitterQuotedMessage.java │ │ │ │ │ └── TwitterRetweetMessage.java │ │ │ │ ├── features/ │ │ │ │ │ ├── AgeDecay.java │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── EarlybirdDocumentFeatures.java │ │ │ │ │ ├── FeatureSink.java │ │ │ │ │ ├── IntNormalizers.java │ │ │ │ │ ├── MutableFeatureNormalizers.java │ │ │ │ │ ├── QueryFeatureType.java │ │ │ │ │ ├── RelevanceSignalConstants.java │ │ │ │ │ ├── ScoringUtils.java │ │ │ │ │ ├── TermVector.java │ │ │ │ │ ├── TweetEngagementFeatures.java │ │ │ │ │ ├── TweetFeatureType.java │ │ │ │ │ ├── TweetFeatures.java │ │ │ │ │ ├── TweetIntegerShingleSignature.java │ │ │ │ │ ├── TweetSignatureUtil.java │ │ │ │ │ ├── TweetTextFeatures.java │ │ │ │ │ ├── TweetTextQuality.java │ │ │ │ │ └── TweetUserFeatures.java │ │ │ │ ├── scorers/ │ │ │ │ │ ├── TweetScorer.java │ │ │ │ │ └── TweetTextScorer.java │ │ │ │ └── text/ │ │ │ │ ├── LocationUtils.java │ │ │ │ ├── TweetParser.java │ │ │ │ └── VisibleTokenRatioNormalizer.java │ │ │ ├── schema/ │ │ │ │ ├── AnalyzerFactory.java │ │ │ │ ├── BUILD │ │ │ │ ├── DynamicSchema.java │ │ │ │ ├── ImmutableSchema.java │ │ │ │ ├── NumericField.java │ │ │ │ ├── SchemaBuilder.java │ │ │ │ ├── SchemaDocumentFactory.java │ │ │ │ ├── SchemaUtil.java │ │ │ │ ├── SearchWhitespaceAnalyzer.java │ │ │ │ ├── ThriftDocumentBuilder.java │ │ │ │ ├── base/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── EarlybirdFieldType.java │ │ │ │ │ ├── FeatureConfiguration.java │ │ │ │ │ ├── FieldNameToIdMapping.java │ │ │ │ │ ├── FieldWeightDefault.java │ │ │ │ │ ├── ImmutableSchemaInterface.java │ │ │ │ │ ├── IndexedNumericFieldSettings.java │ │ │ │ │ ├── Schema.java │ │ │ │ │ └── ThriftDocumentUtil.java │ │ │ │ └── earlybird/ │ │ │ │ ├── BUILD │ │ │ │ ├── EarlybirdCluster.java │ │ │ │ ├── EarlybirdEncodedFeatures.java │ │ │ │ ├── EarlybirdEncodedFeaturesUtil.java │ │ │ │ ├── EarlybirdFieldConstants.java │ │ │ │ ├── EarlybirdSchemaBuilder.java │ │ │ │ ├── EarlybirdSchemaCreateTool.java │ │ │ │ ├── EarlybirdThriftDocumentBuilder.java │ │ │ │ ├── EarlybirdThriftDocumentUtil.java │ │ │ │ └── FlushVersion.java │ │ │ ├── search/ │ │ │ │ ├── AndNotDocIdSetIterator.java │ │ │ │ ├── BUILD │ │ │ │ ├── DelegatingEarlyTerminationCollector.java │ │ │ │ ├── DocIdTracker.java │ │ │ │ ├── EarlyTerminationState.java │ │ │ │ ├── GeoQuadTreeQueryBuilderUtil.java │ │ │ │ ├── IntArrayDocIdSetIterator.java │ │ │ │ ├── PairDocIdSetIterator.java │ │ │ │ ├── QueryCostProvider.java │ │ │ │ ├── TerminationTracker.java │ │ │ │ ├── TwitterCollector.java │ │ │ │ ├── TwitterEarlyTerminationCollector.java │ │ │ │ ├── TwitterIndexSearcher.java │ │ │ │ └── termination/ │ │ │ │ ├── BUILD │ │ │ │ ├── QueryTimeout.java │ │ │ │ ├── QueryTimeoutFactory.java │ │ │ │ ├── QueryTimeoutImpl.java │ │ │ │ ├── TerminationQuery.java │ │ │ │ ├── TerminationQueryScorer.java │ │ │ │ └── TerminationQueryWeight.java │ │ │ └── util/ │ │ │ ├── earlybird/ │ │ │ │ ├── BUILD │ │ │ │ ├── EarlybirdResponseMergeUtil.java │ │ │ │ ├── EarlybirdResponseUtil.java │ │ │ │ ├── FacetsResultsUtils.java │ │ │ │ ├── ResponseMergerUtils.java │ │ │ │ ├── ResultsUtil.java │ │ │ │ ├── TermStatisticsUtil.java │ │ │ │ ├── ThriftSearchQueryUtil.java │ │ │ │ ├── ThriftSearchResultUtil.java │ │ │ │ └── ThriftSearchResultsRelevanceStatsUtil.java │ │ │ ├── lang/ │ │ │ │ ├── BUILD │ │ │ │ └── ThriftLanguageUtil.java │ │ │ └── ml/ │ │ │ ├── BUILD │ │ │ ├── EnumBasedLinearModel.java │ │ │ ├── FeatureUtils.java │ │ │ ├── MapBasedLinearModel.java │ │ │ ├── StringMapBasedLinearModel.java │ │ │ ├── models_manager/ │ │ │ │ ├── BUILD │ │ │ │ └── BaseModelsManager.java │ │ │ ├── prediction_engine/ │ │ │ │ ├── BUILD │ │ │ │ ├── BaseLegacyScoreAccumulator.java │ │ │ │ ├── BaseModelBuilder.java │ │ │ │ ├── BaseScoreAccumulator.java │ │ │ │ ├── CompositeFeatureContext.java │ │ │ │ ├── DecisionForestModelsManager.java │ │ │ │ ├── DiscretizedFeature.java │ │ │ │ ├── DiscretizedFeatureRange.java │ │ │ │ ├── LegacyModelBuilder.java │ │ │ │ ├── LightweightLinearModel.java │ │ │ │ ├── ModelBuilder.java │ │ │ │ ├── ModelLoader.java │ │ │ │ ├── PredictionEngineModelsManager.java │ │ │ │ ├── SchemaBasedModelBuilder.java │ │ │ │ └── SchemaBasedScoreAccumulator.java │ │ │ └── tensorflow_engine/ │ │ │ ├── BUILD │ │ │ └── TensorflowModelsManager.java │ │ ├── core/ │ │ │ └── earlybird/ │ │ │ ├── BUILD │ │ │ ├── README.md │ │ │ ├── facets/ │ │ │ │ ├── AbstractFacetCountingArray.java │ │ │ │ ├── CSFFacetCountIterator.java │ │ │ │ ├── CompositeFacetCountIterator.java │ │ │ │ ├── DummyFacetAccumulator.java │ │ │ │ ├── EarlybirdFacetDocValueSet.java │ │ │ │ ├── EarlybirdFacets.java │ │ │ │ ├── EarlybirdFacetsFactory.java │ │ │ │ ├── FacetAccumulator.java │ │ │ │ ├── FacetCountAggregator.java │ │ │ │ ├── FacetCountIterator.java │ │ │ │ ├── FacetCountIteratorFactory.java │ │ │ │ ├── FacetCountState.java │ │ │ │ ├── FacetCountingArray.java │ │ │ │ ├── FacetCountingArrayWriter.java │ │ │ │ ├── FacetIDMap.java │ │ │ │ ├── FacetLabelProvider.java │ │ │ │ ├── FacetResponseRewriter.java │ │ │ │ ├── FacetTermCollector.java │ │ │ │ ├── FacetUtil.java │ │ │ │ ├── LanguageHistogram.java │ │ │ │ ├── OptimizedFacetCountingArray.java │ │ │ │ ├── PerfieldFacetCountAggregator.java │ │ │ │ ├── SortedSetDocValuesFacetsFactory.java │ │ │ │ └── SortedSetDocValuesReaderStateHelper.java │ │ │ └── index/ │ │ │ ├── DocIDToTweetIDMapper.java │ │ │ ├── EarlybirdIndexSegmentAtomicReader.java │ │ │ ├── EarlybirdIndexSegmentData.java │ │ │ ├── EarlybirdIndexSegmentWriter.java │ │ │ ├── EarlybirdIndexableField.java │ │ │ ├── EarlybirdLuceneIndexSegmentAtomicReader.java │ │ │ ├── EarlybirdLuceneIndexSegmentData.java │ │ │ ├── EarlybirdLuceneIndexSegmentWriter.java │ │ │ ├── EarlybirdRealtimeIndexSegmentAtomicReader.java │ │ │ ├── EarlybirdRealtimeIndexSegmentData.java │ │ │ ├── EarlybirdRealtimeIndexSegmentWriter.java │ │ │ ├── QueryCacheResultForSegment.java │ │ │ ├── SequentialDocIDMapper.java │ │ │ ├── TimeMapper.java │ │ │ ├── column/ │ │ │ │ ├── AbstractColumnStrideMultiIntIndex.java │ │ │ │ ├── ColumnStrideByteIndex.java │ │ │ │ ├── ColumnStrideFieldDocValues.java │ │ │ │ ├── ColumnStrideFieldIndex.java │ │ │ │ ├── ColumnStrideIntIndex.java │ │ │ │ ├── ColumnStrideIntViewIndex.java │ │ │ │ ├── ColumnStrideLongIndex.java │ │ │ │ ├── ColumnStrideMultiIntIndex.java │ │ │ │ ├── ConstantColumnStrideFieldIndex.java │ │ │ │ ├── DocValuesManager.java │ │ │ │ ├── DocValuesUpdate.java │ │ │ │ ├── OptimizedColumnStrideByteIndex.java │ │ │ │ ├── OptimizedColumnStrideIntIndex.java │ │ │ │ ├── OptimizedColumnStrideLongIndex.java │ │ │ │ ├── OptimizedColumnStrideMultiIntIndex.java │ │ │ │ ├── OptimizedDocValuesManager.java │ │ │ │ └── UnoptimizedDocValuesManager.java │ │ │ ├── extensions/ │ │ │ │ ├── EarlybirdIndexExtensionsData.java │ │ │ │ ├── EarlybirdIndexExtensionsFactory.java │ │ │ │ └── EarlybirdRealtimeIndexExtensionsData.java │ │ │ ├── inverted/ │ │ │ │ ├── BaseByteBlockPool.java │ │ │ │ ├── ByteBlockPool.java │ │ │ │ ├── ByteTermUtils.java │ │ │ │ ├── DeletedDocs.java │ │ │ │ ├── EarlybirdCSFDocValuesProcessor.java │ │ │ │ ├── EarlybirdOptimizedPostingsEnum.java │ │ │ │ ├── EarlybirdPostingsEnum.java │ │ │ │ ├── FSTTermDictionary.java │ │ │ │ ├── HighDFPackedIntsDocsAndPositionsEnum.java │ │ │ │ ├── HighDFPackedIntsDocsEnum.java │ │ │ │ ├── HighDFPackedIntsPostingLists.java │ │ │ │ ├── HighDFPackedIntsSkipListReader.java │ │ │ │ ├── InMemoryFields.java │ │ │ │ ├── IndexOptimizer.java │ │ │ │ ├── IntBlockPool.java │ │ │ │ ├── IntBlockPoolPackedLongsReader.java │ │ │ │ ├── IntBlockPoolPackedLongsWriter.java │ │ │ │ ├── InvertedIndex.java │ │ │ │ ├── InvertedRealtimeIndex.java │ │ │ │ ├── InvertedRealtimeIndexWriter.java │ │ │ │ ├── LowDFPackedIntsPostingLists.java │ │ │ │ ├── LowDFPackedIntsPostingsEnum.java │ │ │ │ ├── MPHTermDictionary.java │ │ │ │ ├── MultiPostingLists.java │ │ │ │ ├── MultiSegmentTermDictionary.java │ │ │ │ ├── MultiSegmentTermDictionaryWithFastutil.java │ │ │ │ ├── MultiSegmentTermDictionaryWithMap.java │ │ │ │ ├── OptimizedIndexTerms.java │ │ │ │ ├── OptimizedMemoryIndex.java │ │ │ │ ├── OptimizedPostingLists.java │ │ │ │ ├── OptimizingPostingsEnumWrapper.java │ │ │ │ ├── PackedLongsReaderPreComputedValues.java │ │ │ │ ├── PayloadUtil.java │ │ │ │ ├── PostingsBufferQueue.java │ │ │ │ ├── QueryCostTracker.java │ │ │ │ ├── RealtimeIndexTerms.java │ │ │ │ ├── SkipListComparator.java │ │ │ │ ├── SkipListContainer.java │ │ │ │ ├── SkipListIntegerComparator.java │ │ │ │ ├── SkipListPostingList.java │ │ │ │ ├── SkipListPostingsEnum.java │ │ │ │ ├── SkipListSearchFinger.java │ │ │ │ ├── TermDictionary.java │ │ │ │ ├── TermPointerEncoding.java │ │ │ │ └── TermsArray.java │ │ │ └── util/ │ │ │ ├── AllDocsIterator.java │ │ │ ├── RangeDISI.java │ │ │ ├── RangeFilterDISI.java │ │ │ └── SearchSortUtils.java │ │ ├── earlybird/ │ │ │ ├── BUILD │ │ │ ├── CONFIG.ini │ │ │ ├── Earlybird.java │ │ │ ├── EarlybirdCPUQualityFactor.java │ │ │ ├── EarlybirdDarkProxy.java │ │ │ ├── EarlybirdFinagleServerManager.java │ │ │ ├── EarlybirdFuturePoolManager.java │ │ │ ├── EarlybirdIndexConfig.java │ │ │ ├── EarlybirdMain.java │ │ │ ├── EarlybirdProductionFinagleServerManager.java │ │ │ ├── EarlybirdSearcher.java │ │ │ ├── EarlybirdServer.java │ │ │ ├── EarlybirdServerSetManager.java │ │ │ ├── EarlybirdStatus.java │ │ │ ├── EarlybirdWarmUpManager.java │ │ │ ├── QualityFactor.java │ │ │ ├── README.md │ │ │ ├── RealtimeEarlybirdIndexConfig.java │ │ │ ├── RecentTweetRestriction.java │ │ │ ├── ServerSetMember.java │ │ │ ├── UpdateableEarlybirdStateManager.java │ │ │ ├── archive/ │ │ │ │ ├── ArchiveEarlybirdIndexConfig.java │ │ │ │ ├── ArchiveHDFSUtils.java │ │ │ │ ├── ArchiveOnDiskEarlybirdIndexConfig.java │ │ │ │ ├── ArchiveSearchPartitionManager.java │ │ │ │ ├── ArchiveSegment.java │ │ │ │ ├── ArchiveSegmentDataProvider.java │ │ │ │ ├── ArchiveSegmentUpdater.java │ │ │ │ ├── ArchiveSegmentVerifier.java │ │ │ │ ├── ArchiveTimeSlicer.java │ │ │ │ ├── DailyStatusBatch.java │ │ │ │ ├── DailyStatusBatches.java │ │ │ │ ├── PartitionedBatch.java │ │ │ │ └── segmentbuilder/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── BuiltAndFinalizedSegment.java │ │ │ │ ├── NotYetBuiltSegment.java │ │ │ │ ├── RateLimitingSegmentHandler.java │ │ │ │ ├── SegmentBuilder.java │ │ │ │ ├── SegmentBuilderApp.java │ │ │ │ ├── SegmentBuilderCoordinator.java │ │ │ │ ├── SegmentBuilderMain.java │ │ │ │ ├── SegmentBuilderModule.java │ │ │ │ ├── SegmentBuilderSegment.java │ │ │ │ ├── SegmentConfig.java │ │ │ │ ├── SegmentInfoConstructionException.java │ │ │ │ ├── SegmentUpdaterException.java │ │ │ │ └── SomeoneElseIsBuildingSegment.java │ │ │ ├── common/ │ │ │ │ ├── BUILD │ │ │ │ ├── Base64RequestResponseForLogging.java │ │ │ │ ├── CaughtUpMonitor.java │ │ │ │ ├── ClientIdUtil.java │ │ │ │ ├── EarlybirdRequestLogger.java │ │ │ │ ├── EarlybirdRequestPostLogger.java │ │ │ │ ├── EarlybirdRequestPreLogger.java │ │ │ │ ├── EarlybirdRequestUtil.java │ │ │ │ ├── EarlybirdThriftBackend.java │ │ │ │ ├── NonPagingAssert.java │ │ │ │ ├── RequestResponseForLogging.java │ │ │ │ ├── RequestResponsePair.java │ │ │ │ ├── UnknownClientRequestForLogging.java │ │ │ │ ├── config/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── EarlybirdConfig.java │ │ │ │ │ └── EarlybirdProperty.java │ │ │ │ └── userupdates/ │ │ │ │ ├── BUILD │ │ │ │ ├── UserScrubGeoMap.java │ │ │ │ ├── UserTable.java │ │ │ │ ├── UserTableBuilderFromSnapshot.java │ │ │ │ ├── UserUpdate.java │ │ │ │ └── UserUpdatesChecker.java │ │ │ ├── config/ │ │ │ │ ├── BUILD │ │ │ │ ├── ServingRange.java │ │ │ │ ├── TierConfig.java │ │ │ │ ├── TierInfo.java │ │ │ │ ├── TierInfoSource.java │ │ │ │ ├── TierInfoUtil.java │ │ │ │ ├── TierInfoWrapper.java │ │ │ │ └── TierServingBoundaryEndPoint.java │ │ │ ├── document/ │ │ │ │ ├── DeletedStatus.java │ │ │ │ ├── DocumentFactory.java │ │ │ │ ├── ThriftDocumentPreprocessor.java │ │ │ │ ├── ThriftIndexingEventDocumentFactory.java │ │ │ │ ├── ThriftIndexingEventUpdateFactory.java │ │ │ │ ├── TimeSlicedThriftIndexingEvent.java │ │ │ │ ├── TruncationTokenStreamWriter.java │ │ │ │ └── TweetDocument.java │ │ │ ├── exception/ │ │ │ │ ├── AlreadyInServerSetUpdateException.java │ │ │ │ ├── BadRequestException.java │ │ │ │ ├── ClientException.java │ │ │ │ ├── CriticalExceptionHandler.java │ │ │ │ ├── EarlybirdException.java │ │ │ │ ├── EarlybirdFinagleServerMonitor.java │ │ │ │ ├── EarlybirdRuntimeException.java │ │ │ │ ├── EarlybirdStartupException.java │ │ │ │ ├── FlushVersionMismatchException.java │ │ │ │ ├── MissingKafkaTopicException.java │ │ │ │ ├── MissingUserException.java │ │ │ │ ├── NotInServerSetUpdateException.java │ │ │ │ ├── TransientException.java │ │ │ │ ├── UncaughtExceptionHandler.java │ │ │ │ └── WrappedKafkaApiException.java │ │ │ ├── factory/ │ │ │ │ ├── EarlybirdIndexConfigUtil.java │ │ │ │ ├── EarlybirdKafkaConsumersFactory.java │ │ │ │ ├── EarlybirdServerFactory.java │ │ │ │ ├── EarlybirdWireModule.java │ │ │ │ ├── PartitionConfigUtil.java │ │ │ │ ├── ProductionEarlybirdKafkaConsumersFactory.java │ │ │ │ └── QueryCacheUpdaterScheduledExecutorService.java │ │ │ ├── index/ │ │ │ │ ├── AbstractInMemoryTimeMapper.java │ │ │ │ ├── DocValuesBasedTimeMapper.java │ │ │ │ ├── DocValuesBasedTweetIDMapper.java │ │ │ │ ├── DocValuesHelper.java │ │ │ │ ├── EarlybirdSegment.java │ │ │ │ ├── EarlybirdSegmentFactory.java │ │ │ │ ├── EarlybirdSingleSegmentSearcher.java │ │ │ │ ├── OptimizedTimeMapper.java │ │ │ │ ├── OptimizedTweetIDMapper.java │ │ │ │ ├── OutOfOrderRealtimeTweetIDMapper.java │ │ │ │ ├── RealtimeTimeMapper.java │ │ │ │ ├── TimeMappingWriter.java │ │ │ │ ├── TweetIDMapper.java │ │ │ │ ├── TweetIDQuery.java │ │ │ │ ├── TweetIDToInternalIDMap.java │ │ │ │ ├── TweetSearchIndexExtensionsFactory.java │ │ │ │ ├── TweetSearchLuceneIndexExtensionsData.java │ │ │ │ ├── TweetSearchRealtimeIndexExtensionsData.java │ │ │ │ └── facets/ │ │ │ │ ├── BUILD │ │ │ │ └── FacetSkipList.java │ │ │ ├── ml/ │ │ │ │ └── ScoringModelsManager.java │ │ │ ├── partition/ │ │ │ │ ├── AudioSpaceEventsStreamIndexer.java │ │ │ │ ├── AudioSpaceTable.java │ │ │ │ ├── BalancingKafkaConsumer.java │ │ │ │ ├── CompleteSegmentManager.java │ │ │ │ ├── DynamicPartitionConfig.java │ │ │ │ ├── EarlybirdIndex.java │ │ │ │ ├── EarlybirdIndexFlusher.java │ │ │ │ ├── EarlybirdIndexLoader.java │ │ │ │ ├── EarlybirdKafkaConsumer.java │ │ │ │ ├── EarlybirdStartup.java │ │ │ │ ├── FlowControlException.java │ │ │ │ ├── HdfsUtil.java │ │ │ │ ├── ISegmentWriter.java │ │ │ │ ├── IndexingResultCounts.java │ │ │ │ ├── InstrumentedQueue.java │ │ │ │ ├── KafkaStartup.java │ │ │ │ ├── MultiSegmentTermDictionaryManager.java │ │ │ │ ├── OptimizationAndFlushingCoordinationLock.java │ │ │ │ ├── OptimizingSegmentWriter.java │ │ │ │ ├── PartitionConfig.java │ │ │ │ ├── PartitionConfigLoader.java │ │ │ │ ├── PartitionConfigLoadingException.java │ │ │ │ ├── PartitionManager.java │ │ │ │ ├── PartitionManagerStartup.java │ │ │ │ ├── PartitionWriter.java │ │ │ │ ├── SearchIndexingMetricSet.java │ │ │ │ ├── SegmentHdfsFlusher.java │ │ │ │ ├── SegmentIndexStats.java │ │ │ │ ├── SegmentIndexStatsExporter.java │ │ │ │ ├── SegmentInfo.java │ │ │ │ ├── SegmentLoader.java │ │ │ │ ├── SegmentManager.java │ │ │ │ ├── SegmentOptimizer.java │ │ │ │ ├── SegmentSyncConfig.java │ │ │ │ ├── SegmentSyncInfo.java │ │ │ │ ├── SegmentVulture.java │ │ │ │ ├── SegmentWarmer.java │ │ │ │ ├── SegmentWriter.java │ │ │ │ ├── SimpleSegmentIndexer.java │ │ │ │ ├── SimpleStreamIndexer.java │ │ │ │ ├── SimpleUpdateIndexer.java │ │ │ │ ├── StartupUserEventIndexer.java │ │ │ │ ├── StatusBatchFlushVersion.java │ │ │ │ ├── TimeLimitedHadoopExistsCall.java │ │ │ │ ├── TweetCreateHandler.java │ │ │ │ ├── TweetUpdateHandler.java │ │ │ │ ├── UserPartitionUtil.java │ │ │ │ ├── UserScrubGeoEventStreamIndexer.java │ │ │ │ ├── UserUpdatesStreamIndexer.java │ │ │ │ └── freshstartup/ │ │ │ │ ├── FreshStartupHandler.java │ │ │ │ ├── KafkaOffsetPair.java │ │ │ │ ├── PostOptimizationUpdatesIndexer.java │ │ │ │ ├── PreOptimizationSegmentIndexer.java │ │ │ │ ├── SegmentBuildInfo.java │ │ │ │ ├── SegmentTweetsIndexingResult.java │ │ │ │ └── SkippedPickedCounter.java │ │ │ ├── querycache/ │ │ │ │ ├── CachedFilterQuery.java │ │ │ │ ├── CachedResultDocIdSetIterator.java │ │ │ │ ├── QueryCacheConfig.java │ │ │ │ ├── QueryCacheConversionRules.java │ │ │ │ ├── QueryCacheFilter.java │ │ │ │ ├── QueryCacheManager.java │ │ │ │ ├── QueryCacheResultCollector.java │ │ │ │ ├── QueryCacheUpdateTask.java │ │ │ │ └── QueryCacheUpdater.java │ │ │ ├── queryparser/ │ │ │ │ ├── DetectAntisocialVisitor.java │ │ │ │ ├── DetectFieldAnnotationVisitor.java │ │ │ │ ├── EarlybirdLuceneQueryVisitor.java │ │ │ │ ├── EarlybirdQueryHelper.java │ │ │ │ ├── HighFrequencyTermPairExtractor.java │ │ │ │ ├── HighFrequencyTermPairRewriteVisitor.java │ │ │ │ ├── HighFrequencyTermQueryGroup.java │ │ │ │ ├── LuceneRelevanceQueryVisitor.java │ │ │ │ └── ProtectedOperatorQueryRewriter.java │ │ │ ├── search/ │ │ │ │ ├── AbstractResultsCollector.java │ │ │ │ ├── AntiGamingFilter.java │ │ │ │ ├── EarlybirdLuceneSearcher.java │ │ │ │ ├── EarlybirdMultiSegmentSearcher.java │ │ │ │ ├── GeoQuadTreeQueryBuilder.java │ │ │ │ ├── Hit.java │ │ │ │ ├── SearchRequestInfo.java │ │ │ │ ├── SearchResultsCollector.java │ │ │ │ ├── SearchResultsInfo.java │ │ │ │ ├── SimpleSearchResults.java │ │ │ │ ├── SocialFilter.java │ │ │ │ ├── SocialSearchResultsCollector.java │ │ │ │ ├── facets/ │ │ │ │ │ ├── AbstractFacetTermCollector.java │ │ │ │ │ ├── DefaultFacetScorer.java │ │ │ │ │ ├── EntityAnnotationCollector.java │ │ │ │ │ ├── ExpandedUrlCollector.java │ │ │ │ │ ├── ExplainFacetResultsCollector.java │ │ │ │ │ ├── FacetLabelCollector.java │ │ │ │ │ ├── FacetRankingModule.java │ │ │ │ │ ├── FacetResultsCollector.java │ │ │ │ │ ├── FacetScorer.java │ │ │ │ │ ├── FacetSearchRequestInfo.java │ │ │ │ │ ├── HashingAndPruningFacetAccumulator.java │ │ │ │ │ ├── NamedEntityCollector.java │ │ │ │ │ ├── RetweetFacetCountIterator.java │ │ │ │ │ ├── SimpleCountRankingModule.java │ │ │ │ │ ├── SpaceFacetCollector.java │ │ │ │ │ ├── TermStatisticsCollector.java │ │ │ │ │ ├── TermStatisticsRequestInfo.java │ │ │ │ │ └── TweetSearchFacetCountIteratorFactory.java │ │ │ │ ├── queries/ │ │ │ │ │ ├── BadUserRepFilter.java │ │ │ │ │ ├── CSFDisjunctionFilter.java │ │ │ │ │ ├── DocValRangeFilter.java │ │ │ │ │ ├── FeatureValueInAcceptListOrUnsetFilter.java │ │ │ │ │ ├── GeoTwoPhaseQuery.java │ │ │ │ │ ├── MatchAllDocIdSet.java │ │ │ │ │ ├── MatchAllDocsQuery.java │ │ │ │ │ ├── RequiredStatusIDsFilter.java │ │ │ │ │ ├── SimpleTermQuery.java │ │ │ │ │ ├── SinceMaxIDFilter.java │ │ │ │ │ ├── SinceUntilFilter.java │ │ │ │ │ ├── TermQueryWithSafeToString.java │ │ │ │ │ ├── TimedDocIdSetIterator.java │ │ │ │ │ ├── UserFlagsExcludeFilter.java │ │ │ │ │ ├── UserIdMultiSegmentQuery.java │ │ │ │ │ └── UserScrubGeoFilter.java │ │ │ │ └── relevance/ │ │ │ │ ├── LinearScoringData.java │ │ │ │ ├── LinearScoringParams.java │ │ │ │ ├── MinFeatureValueFilter.java │ │ │ │ ├── RelevanceHit.java │ │ │ │ ├── RelevanceSearchRequestInfo.java │ │ │ │ ├── RelevanceSearchResults.java │ │ │ │ ├── ScoreFilterQuery.java │ │ │ │ ├── collectors/ │ │ │ │ │ ├── AbstractRelevanceCollector.java │ │ │ │ │ ├── BatchRelevanceTopCollector.java │ │ │ │ │ ├── RelevanceAllCollector.java │ │ │ │ │ └── RelevanceTopCollector.java │ │ │ │ └── scoring/ │ │ │ │ ├── BatchHit.java │ │ │ │ ├── DefaultScoringFunction.java │ │ │ │ ├── FeatureBasedScoringFunction.java │ │ │ │ ├── LegacyScoreAccumulator.java │ │ │ │ ├── LinearScoringFunction.java │ │ │ │ ├── ModelBasedScoringFunction.java │ │ │ │ ├── RelevanceQuery.java │ │ │ │ ├── RetweetBasedTopTweetsScoringFunction.java │ │ │ │ ├── ScoringFunction.java │ │ │ │ ├── ScoringFunctionProvider.java │ │ │ │ ├── SpamVectorScoringFunction.java │ │ │ │ ├── SparseTensor.java │ │ │ │ ├── TensorflowBasedScoringFunction.java │ │ │ │ └── TestScoringFunction.java │ │ │ ├── segment/ │ │ │ │ ├── DLSegmentDataProvider.java │ │ │ │ ├── DLSegmentDataReaderSet.java │ │ │ │ ├── EmptySegmentDataReaderSet.java │ │ │ │ ├── SegmentDataProvider.java │ │ │ │ ├── SegmentDataReaderSet.java │ │ │ │ └── SegmentProvider.java │ │ │ ├── stats/ │ │ │ │ ├── EarlybirdRPCStats.java │ │ │ │ ├── EarlybirdSearcherStats.java │ │ │ │ └── SegmentSyncStats.java │ │ │ ├── tools/ │ │ │ │ └── EarlybirdThriftRequestDeserializerUtil.java │ │ │ └── util/ │ │ │ ├── ActionLogger.java │ │ │ ├── CoordinatedEarlybirdAction.java │ │ │ ├── CoordinatedEarlybirdActionInterface.java │ │ │ ├── CoordinatedEarlybirdActionLockFailed.java │ │ │ ├── EarlybirdDecider.java │ │ │ ├── EarlybirdSearchResultUtil.java │ │ │ ├── FieldTermCounter.java │ │ │ ├── Histogram.java │ │ │ ├── IndexViewer.java │ │ │ ├── JsonViewerWriter.java │ │ │ ├── OneTaskScheduledExecutorManager.java │ │ │ ├── ParallelUtil.java │ │ │ ├── PeriodicActionParams.java │ │ │ ├── ScheduledExecutorManager.java │ │ │ ├── ScheduledExecutorTask.java │ │ │ ├── ScrubGenUtil.java │ │ │ ├── ShutdownWaitTimeParams.java │ │ │ ├── TermCountMonitor.java │ │ │ ├── TweetCountMonitor.java │ │ │ └── ViewerWriter.java │ │ ├── earlybird_root/ │ │ │ ├── BUILD │ │ │ ├── ClientBackupFilter.java │ │ │ ├── ClientLatencyFilter.java │ │ │ ├── EarlybirdCacheCommonModule.java │ │ │ ├── EarlybirdChainedScatterGatherService.java │ │ │ ├── EarlybirdCommonModule.java │ │ │ ├── EarlybirdFullArchiveScatterGatherSupport.java │ │ │ ├── EarlybirdProtectedScatterGatherSupport.java │ │ │ ├── EarlybirdProtectedValidationBehavior.java │ │ │ ├── EarlybirdProtectedWarmup.java │ │ │ ├── EarlybirdQueryRewriteFilter.java │ │ │ ├── EarlybirdRealtimeCgScatterGatherSupport.java │ │ │ ├── EarlybirdRealtimeScatterGatherSupport.java │ │ │ ├── EarlybirdRootQueryUtils.java │ │ │ ├── EarlybirdServiceChainBuilder.java │ │ │ ├── EarlybirdServiceLoggingSupport.java │ │ │ ├── EarlybirdServicePartitionLoggingSupport.java │ │ │ ├── EarlybirdServiceScatterGatherSupport.java │ │ │ ├── EarlybirdServiceValidationBehavior.java │ │ │ ├── EarlybirdTierThrottleDeciders.java │ │ │ ├── EarlybirdWarmup.java │ │ │ ├── ExceptionHandler.java │ │ │ ├── FullArchiveRootAppMain.java │ │ │ ├── FullArchiveRootModule.java │ │ │ ├── FullArchiveRootServer.java │ │ │ ├── FullArchiveRootService.java │ │ │ ├── InitializeFilter.java │ │ │ ├── MultiTierResultsMergeFilter.java │ │ │ ├── PartitionAccessController.java │ │ │ ├── ProtectedRootAppMain.java │ │ │ ├── ProtectedRootAppModule.java │ │ │ ├── ProtectedRootServer.java │ │ │ ├── ProtectedRootService.java │ │ │ ├── ProtectedScatterGatherModule.java │ │ │ ├── QuotaModule.java │ │ │ ├── README.md │ │ │ ├── RealtimeCgRootAppMain.java │ │ │ ├── RealtimeCgRootAppModule.java │ │ │ ├── RealtimeCgRootServer.java │ │ │ ├── RealtimeCgRootService.java │ │ │ ├── RealtimeCgScatterGatherModule.java │ │ │ ├── RealtimeRootAppMain.java │ │ │ ├── RealtimeRootAppModule.java │ │ │ ├── RealtimeRootServer.java │ │ │ ├── RealtimeRootService.java │ │ │ ├── RealtimeScatterGatherModule.java │ │ │ ├── RootResponseClassifier.java │ │ │ ├── ScatterGatherModule.java │ │ │ ├── SkipPartitionFilter.java │ │ │ ├── SuperRootAppMain.java │ │ │ ├── SuperRootAppModule.java │ │ │ ├── SuperRootRequestTypeRouter.java │ │ │ ├── SuperRootServer.java │ │ │ ├── SuperRootService.java │ │ │ ├── caching/ │ │ │ │ ├── BUILD │ │ │ │ ├── CacheCommonUtil.java │ │ │ │ ├── CacheStats.java │ │ │ │ ├── DefaultForcedCacheMissDecider.java │ │ │ │ ├── EarlybirdCachePostProcessor.java │ │ │ │ ├── EarlybirdRequestPerClientCacheStats.java │ │ │ │ ├── FacetsCache.java │ │ │ │ ├── FacetsCacheFilter.java │ │ │ │ ├── FacetsCacheRequestNormalizer.java │ │ │ │ ├── FacetsQueryCachePredicate.java │ │ │ │ ├── FacetsServicePostProcessor.java │ │ │ │ ├── RecencyAndRelevanceCachePostProcessor.java │ │ │ │ ├── RecencyCache.java │ │ │ │ ├── RecencyCacheFilter.java │ │ │ │ ├── RecencyCacheRequestNormalizer.java │ │ │ │ ├── RecencyQueryCachePredicate.java │ │ │ │ ├── RecencyServicePostProcessor.java │ │ │ │ ├── RelevanceCache.java │ │ │ │ ├── RelevanceCacheFilter.java │ │ │ │ ├── RelevanceCacheRequestNormalizer.java │ │ │ │ ├── RelevanceQueryCachePredicate.java │ │ │ │ ├── RelevanceServicePostProcessor.java │ │ │ │ ├── RelevanceZeroResultsCacheFilter.java │ │ │ │ ├── RelevanceZeroResultsCachePostProcessor.java │ │ │ │ ├── RelevanceZeroResultsCacheRequestNormalizer.java │ │ │ │ ├── RelevanceZeroResultsQueryCachePredicate.java │ │ │ │ ├── RelevanceZeroResultsServicePostProcessor.java │ │ │ │ ├── StrictRecencyCache.java │ │ │ │ ├── StrictRecencyCacheFilter.java │ │ │ │ ├── StrictRecencyQueryCachePredicate.java │ │ │ │ ├── TermStatsCache.java │ │ │ │ ├── TermStatsCacheFilter.java │ │ │ │ ├── TermStatsCacheRequestNormalizer.java │ │ │ │ ├── TermStatsQueryCachePredicate.java │ │ │ │ ├── TermStatsServicePostProcessor.java │ │ │ │ ├── TopTweetsCache.java │ │ │ │ ├── TopTweetsCacheFilter.java │ │ │ │ ├── TopTweetsCacheRequestNormalizer.java │ │ │ │ ├── TopTweetsQueryCachePredicate.java │ │ │ │ └── TopTweetsServicePostProcessor.java │ │ │ ├── collectors/ │ │ │ │ ├── BUILD │ │ │ │ ├── MultiwayMergeCollector.java │ │ │ │ ├── RecencyMergeCollector.java │ │ │ │ └── RelevanceMergeCollector.java │ │ │ ├── common/ │ │ │ │ ├── BUILD │ │ │ │ ├── ClientErrorException.java │ │ │ │ ├── EarlybirdFeatureSchemaMerger.java │ │ │ │ ├── EarlybirdRequestContext.java │ │ │ │ ├── EarlybirdRequestType.java │ │ │ │ ├── EarlybirdRequestUtil.java │ │ │ │ ├── EarlybirdServiceResponse.java │ │ │ │ ├── InjectionNames.java │ │ │ │ ├── QueryParsingUtils.java │ │ │ │ └── TwitterContextProvider.java │ │ │ ├── config/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── RootClusterBoundaryInfo.java │ │ │ ├── filters/ │ │ │ │ ├── BUILD │ │ │ │ ├── ClientIdArchiveAccessFilter.java │ │ │ │ ├── ClientIdQueryOperatorStatsFilter.java │ │ │ │ ├── ClientIdQuotaFilter.java │ │ │ │ ├── ClientIdTrackingFilter.java │ │ │ │ ├── ClientRequestTimeFilter.java │ │ │ │ ├── DeadlineTimeoutStatsFilter.java │ │ │ │ ├── DisableClientByTierFilter.java │ │ │ │ ├── DropAllProtectedOperatorFilter.java │ │ │ │ ├── EarlybirdClusterAvailableFilter.java │ │ │ │ ├── EarlybirdFeatureSchemaAnnotateFilter.java │ │ │ │ ├── EarlybirdResponseExceptionHandler.java │ │ │ │ ├── EarlybirdSuccessfulResponseHandler.java │ │ │ │ ├── EarlybirdTimeFilterQueryRewriter.java │ │ │ │ ├── EarlybirdTimeRangeFilter.java │ │ │ │ ├── FullArchiveProtectedOperatorFilter.java │ │ │ │ ├── FullArchiveServingRangeProvider.java │ │ │ │ ├── InitializeRequestContextFilter.java │ │ │ │ ├── IsUserProtectedMetadataTrackingFilter.java │ │ │ │ ├── MarkTweetSourceFilter.java │ │ │ │ ├── MetadataTrackingFilter.java │ │ │ │ ├── NamedMultiTermDisjunctionStatsFilter.java │ │ │ │ ├── NullcastTrackingFilter.java │ │ │ │ ├── PostCacheRequestTypeCountFilter.java │ │ │ │ ├── PreCacheRequestTypeCountFilter.java │ │ │ │ ├── QueryLangStatFilter.java │ │ │ │ ├── QueryOperatorStatFilter.java │ │ │ │ ├── QueryTokenizerFilter.java │ │ │ │ ├── RealtimeServingRangeProvider.java │ │ │ │ ├── RejectRequestsByQuerySourceFilter.java │ │ │ │ ├── RequestContextToEarlybirdRequestFilter.java │ │ │ │ ├── RequestResultStatsFilter.java │ │ │ │ ├── RequestSuccessStatsFilter.java │ │ │ │ ├── RequestTypeCountFilter.java │ │ │ │ ├── ResponseCodeStatFilter.java │ │ │ │ ├── ResultTierCountFilter.java │ │ │ │ ├── ScatterGatherWithExperimentRedirectsService.java │ │ │ │ ├── SearchPayloadSizeLocalContextFilter.java │ │ │ │ ├── SensitiveResultsTrackingFilter.java │ │ │ │ ├── ServiceExceptionHandlingFilter.java │ │ │ │ ├── ServiceResponseValidationFilter.java │ │ │ │ ├── ServingRangeProvider.java │ │ │ │ ├── StratoAttributionClientIdFilter.java │ │ │ │ ├── TopLevelExceptionHandlingFilter.java │ │ │ │ ├── UnsetSuperRootFieldsFilter.java │ │ │ │ └── VeryRecentTweetsFilter.java │ │ │ ├── mergers/ │ │ │ │ ├── AccumulatedResponses.java │ │ │ │ ├── BUILD │ │ │ │ ├── EarlyTerminateTierMergePredicate.java │ │ │ │ ├── EarlybirdResponseDebugMessageBuilder.java │ │ │ │ ├── EarlybirdResponseMerger.java │ │ │ │ ├── FacetResponseMerger.java │ │ │ │ ├── PartitionResponseAccumulator.java │ │ │ │ ├── RecencyResponseMerger.java │ │ │ │ ├── RelevanceResponseMerger.java │ │ │ │ ├── ResponseAccumulator.java │ │ │ │ ├── StrictRecencyResponseMerger.java │ │ │ │ ├── SuperRootResponseMerger.java │ │ │ │ ├── TermStatisticsResponseMerger.java │ │ │ │ ├── ThriftTermResultsMerger.java │ │ │ │ ├── TierResponseAccumulator.java │ │ │ │ ├── TopTweetsResponseMerger.java │ │ │ │ └── TrimStats.java │ │ │ ├── quota/ │ │ │ │ ├── BUILD │ │ │ │ ├── ClientIdQuotaManager.java │ │ │ │ ├── ConfigBasedQuotaConfig.java │ │ │ │ ├── ConfigRepoBasedQuotaManager.java │ │ │ │ └── QuotaInfo.java │ │ │ ├── routers/ │ │ │ │ ├── AbstractRecencyAndRelevanceRequestRouter.java │ │ │ │ ├── BUILD │ │ │ │ ├── FacetsRequestRouter.java │ │ │ │ ├── FacetsRequestRouterModule.java │ │ │ │ ├── RecencyRequestRouter.java │ │ │ │ ├── RecencyRequestRouterModule.java │ │ │ │ ├── RelevanceRequestRouter.java │ │ │ │ ├── RelevanceRequestRouterModule.java │ │ │ │ ├── RequestRouter.java │ │ │ │ ├── RequestRouterUtil.java │ │ │ │ ├── TermStatsRequestRouter.java │ │ │ │ ├── TermStatsRequestRouterModule.java │ │ │ │ ├── TopTweetsRequestRouter.java │ │ │ │ └── TopTweetsRequestRouterModule.java │ │ │ ├── validators/ │ │ │ │ ├── BUILD │ │ │ │ ├── FacetsResponseValidator.java │ │ │ │ ├── PassThroughResponseValidator.java │ │ │ │ ├── SearchResultsValidator.java │ │ │ │ ├── ServiceResponseValidator.java │ │ │ │ ├── TermStatsResultsValidator.java │ │ │ │ └── TopTweetsResultsValidator.java │ │ │ └── visitors/ │ │ │ ├── BUILD │ │ │ └── MultiTermDisjunctionPerPartitionVisitor.java │ │ ├── feature_update_service/ │ │ │ ├── BUILD │ │ │ ├── FeatureUpdateController.java │ │ │ ├── FeatureUpdateResponseClassifier.java │ │ │ ├── FeatureUpdateServiceThriftServer.java │ │ │ ├── FeatureUpdateServiceThriftServerMain.java │ │ │ ├── README.md │ │ │ ├── filters/ │ │ │ │ ├── BUILD │ │ │ │ └── ClientIdWhitelistFilter.java │ │ │ ├── modules/ │ │ │ │ ├── BUILD │ │ │ │ ├── ClientIdWhitelistModule.java │ │ │ │ ├── EarlybirdUtilModule.java │ │ │ │ ├── FeatureUpdateServiceDiffyModule.java │ │ │ │ ├── FinagleKafkaProducerModule.java │ │ │ │ ├── FuturePoolModule.java │ │ │ │ └── TweetypieModule.java │ │ │ ├── stats/ │ │ │ │ ├── BUILD │ │ │ │ └── FeatureUpdateStats.java │ │ │ ├── util/ │ │ │ │ ├── BUILD │ │ │ │ └── FeatureUpdateValidator.java │ │ │ └── whitelist/ │ │ │ ├── BUILD │ │ │ └── ClientIdWhitelist.java │ │ └── ingester/ │ │ ├── BUILD │ │ ├── README.md │ │ ├── model/ │ │ │ ├── BUILD │ │ │ ├── IndexerStatus.java │ │ │ ├── IngesterThriftVersionedEvents.java │ │ │ ├── IngesterTweetEvent.java │ │ │ ├── IngesterTwitterMessage.java │ │ │ ├── KafkaRawRecord.java │ │ │ ├── PromiseContainer.java │ │ │ └── VisibleTokenRatioUtil.java │ │ ├── pipeline/ │ │ │ ├── app/ │ │ │ │ ├── BUILD │ │ │ │ ├── IngesterPipelineApplication.java │ │ │ │ ├── PipelineExceptionImpl.java │ │ │ │ ├── PipelineExceptionImplV2.java │ │ │ │ └── RealtimeIngesterPipelineV2.java │ │ │ ├── strato_fetchers/ │ │ │ │ ├── AudioSpaceCoreFetcher.java │ │ │ │ ├── AudioSpaceParticipantsFetcher.java │ │ │ │ ├── BUILD │ │ │ │ └── NamedEntityFetcher.java │ │ │ ├── twitter/ │ │ │ │ ├── AsyncPinkUrlsResolver.java │ │ │ │ ├── BUILD │ │ │ │ ├── CollectComparableObjectsStage.java │ │ │ │ ├── ComputeTweetSignatureStage.java │ │ │ │ ├── ConvertDelayedMessageToThriftStage.java │ │ │ │ ├── ConvertMessageToThriftStage.java │ │ │ │ ├── ConvertToThriftVersionedEventsStage.java │ │ │ │ ├── EventBusReaderStage.java │ │ │ │ ├── FieldStatExporter.java │ │ │ │ ├── FilterEventsBySafetyTypeStage.java │ │ │ │ ├── FilterRetweetsAndRepliesStage.java │ │ │ │ ├── FilterTwitterMessageStage.java │ │ │ │ ├── LookupUserPropertiesBatchedStage.java │ │ │ │ ├── NamedEntityHandler.java │ │ │ │ ├── PopulateCodedLocationsBatchedStage.java │ │ │ │ ├── ResolveCompressedUrlsBatchedStage.java │ │ │ │ ├── ResolveCompressedUrlsPink.java │ │ │ │ ├── ResolveCompressedUrlsUtils.java │ │ │ │ ├── RetrieveCardBatchedStage.java │ │ │ │ ├── RetrieveNamedEntitiesSingleTweetStage.java │ │ │ │ ├── RetrieveSpaceAdminsAndTitleStage.java │ │ │ │ ├── RetrieveSpaceIdsStage.java │ │ │ │ ├── SingleTweetExtractAndGeocodeLatLonStage.java │ │ │ │ ├── TextFeatureExtractionWorkersStage.java │ │ │ │ ├── TextQualityEvaluationWorkerStage.java │ │ │ │ ├── TextUrlsFeatureExtractionStage.java │ │ │ │ ├── ThriftTweetParserStage.java │ │ │ │ ├── ThriftVersionedEventsConverter.java │ │ │ │ ├── TweetEventDeserializerStage.java │ │ │ │ ├── TwitterBaseStage.java │ │ │ │ ├── TwitterBatchedBaseStage.java │ │ │ │ ├── filters/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── IngesterValidMessageFilter.java │ │ │ │ ├── kafka/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── DeleteUpdateEventsKafkaProducerStage.java │ │ │ │ │ ├── KafkaConsumerStage.java │ │ │ │ │ ├── KafkaProducerStage.java │ │ │ │ │ ├── KafkaRawRecordConsumerStage.java │ │ │ │ │ ├── RetweetAndReplyUpdateEventsKafkaProducerStage.java │ │ │ │ │ └── TweetThriftVersionedEventsKafkaProducerStage.java │ │ │ │ ├── thriftparse/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── ThriftTweetParsingException.java │ │ │ │ │ └── TweetEventParseHelper.java │ │ │ │ └── userupdates/ │ │ │ │ ├── BUILD │ │ │ │ ├── UserUpdateIngester.java │ │ │ │ ├── UserUpdatesPipeline.java │ │ │ │ └── UserUpdatesPipelineStage.java │ │ │ ├── util/ │ │ │ │ ├── BUILD │ │ │ │ ├── BatchedElement.java │ │ │ │ ├── BatchingClient.java │ │ │ │ ├── CardFieldUtil.java │ │ │ │ ├── IngesterStageTimer.java │ │ │ │ ├── ManhattanCodedLocationProvider.java │ │ │ │ ├── PenguinVersionsUtil.java │ │ │ │ ├── PipelineExceptionHandler.java │ │ │ │ ├── PipelineStageException.java │ │ │ │ ├── PipelineStageRuntimeException.java │ │ │ │ ├── PipelineUtil.java │ │ │ │ ├── PipelineV2CreationException.java │ │ │ │ ├── ResponseNotReturnedException.java │ │ │ │ └── UserPropertiesManager.java │ │ │ └── wire/ │ │ │ ├── BUILD │ │ │ ├── IngesterPartitioner.java │ │ │ ├── ProductionWireModule.java │ │ │ ├── StratoMetaStoreWireModule.java │ │ │ ├── TweetyPieWireModule.java │ │ │ └── WireModule.java │ │ └── util/ │ │ └── jndi/ │ │ ├── BUILD │ │ └── JndiUtil.java │ ├── python/ │ │ └── twitter/ │ │ └── deepbird/ │ │ └── projects/ │ │ └── timelines/ │ │ ├── configs/ │ │ │ ├── recap_earlybird/ │ │ │ │ └── feature_config.py │ │ │ └── rectweet_earlybird/ │ │ │ └── feature_config.py │ │ └── scripts/ │ │ └── models/ │ │ └── earlybird/ │ │ ├── BUILD │ │ ├── README.md │ │ ├── __init__.py │ │ ├── constants.py │ │ ├── example_weights.py │ │ ├── lolly/ │ │ │ ├── BUILD │ │ │ ├── __init__.py │ │ │ ├── data_helpers.py │ │ │ ├── parsers.py │ │ │ ├── reader.py │ │ │ ├── score.py │ │ │ ├── scorer.py │ │ │ └── tf_model_initializer_builder.py │ │ ├── metrics.py │ │ ├── tf_model/ │ │ │ ├── BUILD │ │ │ ├── __init__.py │ │ │ ├── discretizer_builder.py │ │ │ ├── hashing_utils.py │ │ │ └── weights_initializer_builder.py │ │ └── train.py │ ├── scala/ │ │ └── com/ │ │ └── twitter/ │ │ ├── graph/ │ │ │ └── batch/ │ │ │ ├── BUILD.bazel │ │ │ └── job/ │ │ │ └── tweepcred/ │ │ │ ├── ExtractTweepcred.scala │ │ │ ├── PreparePageRankData.scala │ │ │ ├── README │ │ │ ├── Reputation.scala │ │ │ ├── TweepcredBatchJob.scala │ │ │ ├── UserMass.scala │ │ │ └── WeightedPageRank.scala │ │ ├── interaction_graph/ │ │ │ ├── README.md │ │ │ ├── bqe/ │ │ │ │ ├── scoring/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── candidates.sql │ │ │ │ │ ├── check_models.sql │ │ │ │ │ ├── follow_graph_features.sql │ │ │ │ │ └── scoring.sql │ │ │ │ └── training/ │ │ │ │ ├── README.md │ │ │ │ ├── candidates.sql │ │ │ │ ├── check_candidates_exist.sql │ │ │ │ ├── check_labels_exist.sql │ │ │ │ ├── labeled_candidates.sql │ │ │ │ └── train_model.sql │ │ │ ├── injection/ │ │ │ │ ├── BUILD │ │ │ │ ├── EdgeListInjection.scala │ │ │ │ └── UserSessionInjection.scala │ │ │ └── scio/ │ │ │ ├── README.md │ │ │ ├── agg_address_book/ │ │ │ │ ├── BUILD │ │ │ │ ├── InteractionGraphAddressBookCounters.scala │ │ │ │ ├── InteractionGraphAddressBookJob.scala │ │ │ │ ├── InteractionGraphAddressBookOption.scala │ │ │ │ ├── InteractionGraphAddressBookSource.scala │ │ │ │ ├── InteractionGraphAddressBookUtil.scala │ │ │ │ └── README.md │ │ │ ├── agg_all/ │ │ │ │ ├── BUILD │ │ │ │ ├── InteractionGraphAggregationConfig.scala │ │ │ │ ├── InteractionGraphAggregationJob.scala │ │ │ │ ├── InteractionGraphAggregationOption.scala │ │ │ │ ├── InteractionGraphAggregationSource.scala │ │ │ │ ├── InteractionGraphAggregationTransform.scala │ │ │ │ └── README.md │ │ │ ├── agg_client_event_logs/ │ │ │ │ ├── BUILD │ │ │ │ ├── InteractionGraphClientEventLogsCounters.scala │ │ │ │ ├── InteractionGraphClientEventLogsJob.scala │ │ │ │ ├── InteractionGraphClientEventLogsOption.scala │ │ │ │ ├── InteractionGraphClientEventLogsSource.scala │ │ │ │ ├── InteractionGraphClientEventLogsUtil.scala │ │ │ │ └── README.md │ │ │ ├── agg_direct_interactions/ │ │ │ │ ├── BUILD │ │ │ │ ├── InteractionGraphAggDirectInteractionsJob.scala │ │ │ │ ├── InteractionGraphAggDirectInteractionsOption.scala │ │ │ │ ├── InteractionGraphAggDirectInteractionsSource.scala │ │ │ │ ├── InteractionGraphAggDirectInteractionsUtil.scala │ │ │ │ └── README.md │ │ │ ├── agg_flock/ │ │ │ │ ├── BUILD │ │ │ │ ├── InteractionGraphAggFlockJob.scala │ │ │ │ ├── InteractionGraphAggFlockOption.scala │ │ │ │ ├── InteractionGraphAggFlockSource.scala │ │ │ │ ├── InteractionGraphAggFlockUtil.scala │ │ │ │ └── README.md │ │ │ ├── agg_negative/ │ │ │ │ ├── BUILD │ │ │ │ ├── InteractionGraphNegativeJob.scala │ │ │ │ ├── InteractionGraphNegativeOption.scala │ │ │ │ └── README.md │ │ │ ├── agg_notifications/ │ │ │ │ ├── BUILD │ │ │ │ ├── InteractionGraphNotificationUtil.scala │ │ │ │ ├── InteractionGraphNotificationsJob.scala │ │ │ │ ├── InteractionGraphNotificationsOption.scala │ │ │ │ └── README.md │ │ │ ├── common/ │ │ │ │ ├── BUILD │ │ │ │ ├── CaseClasses.scala │ │ │ │ ├── ConversionUtil.scala │ │ │ │ ├── DateUtil.scala │ │ │ │ ├── EdgeFeatureCombiner.scala │ │ │ │ ├── FeatureGeneratorUtil.scala │ │ │ │ ├── FeatureGroups.scala │ │ │ │ ├── GraphUtil.scala │ │ │ │ ├── InteractionGraphUtils.scala │ │ │ │ ├── UserUtil.scala │ │ │ │ └── VertexFeatureCombiner.scala │ │ │ └── ml/ │ │ │ ├── labels/ │ │ │ │ ├── BUILD │ │ │ │ ├── InteractionGraphLabelsJob.scala │ │ │ │ ├── InteractionGraphLabelsOption.scala │ │ │ │ ├── LabelUtil.scala │ │ │ │ └── README.md │ │ │ └── scores/ │ │ │ ├── BUILD │ │ │ ├── InteractionGraphScoreExportJob.scala │ │ │ ├── InteractionGraphScoreExportOption.scala │ │ │ └── README.md │ │ ├── recos/ │ │ │ ├── decider/ │ │ │ │ ├── BUILD │ │ │ │ ├── BaseDecider.scala │ │ │ │ └── EndpointLoadShedder.scala │ │ │ ├── graph_common/ │ │ │ │ ├── ActionEdgeTypeMask.scala │ │ │ │ ├── BUILD │ │ │ │ ├── BipartiteGraphHelper.scala │ │ │ │ ├── FinagleCounterWrapper.scala │ │ │ │ ├── FinagleStatsReceiverWrapper.scala │ │ │ │ ├── LeftIndexedPowerLawMultiSegmentBipartiteGraphBuilder.scala │ │ │ │ ├── MultiSegmentPowerLawBipartiteGraphBuilder.scala │ │ │ │ ├── NodeInfoHandler.scala │ │ │ │ ├── NodeMetadataLeftIndexedPowerLawMultiSegmentBipartiteGraphBuilder.scala │ │ │ │ └── RightNodeMetadataLeftIndexedPowerLawMultiSegmentBipartiteGraphBuilder.scala │ │ │ ├── hose/ │ │ │ │ └── common/ │ │ │ │ ├── BUILD │ │ │ │ ├── BufferedEdgeWriter.scala │ │ │ │ ├── EdgeCollector.scala │ │ │ │ ├── RecosEdgeProcessor.scala │ │ │ │ ├── UnifiedGraphWriter.scala │ │ │ │ └── UnifiedGraphWriterMulti.scala │ │ │ ├── user_tweet_entity_graph/ │ │ │ │ ├── BUILD │ │ │ │ ├── EntitySocialProofRunner.scala │ │ │ │ ├── LoggingUserTweetEntityGraph.scala │ │ │ │ ├── Main.scala │ │ │ │ ├── README.md │ │ │ │ ├── RecommendationHandler.scala │ │ │ │ ├── RecosConfig.scala │ │ │ │ ├── SocialProofHandler.scala │ │ │ │ ├── SocialProofHydrator.scala │ │ │ │ ├── TweetRecommendationsRunner.scala │ │ │ │ ├── TweetSocialProofHandler.scala │ │ │ │ ├── TweetSocialProofRunner.scala │ │ │ │ ├── UserTweetEdgeTypeMask.scala │ │ │ │ ├── UserTweetEntityGraph.scala │ │ │ │ └── UserTweetEntityGraphWriter.scala │ │ │ ├── user_tweet_graph/ │ │ │ │ ├── BUILD │ │ │ │ ├── Main.scala │ │ │ │ ├── README.md │ │ │ │ ├── UserTweetGraph.scala │ │ │ │ ├── UserTweetGraphConfig.scala │ │ │ │ ├── UserTweetGraphWriter.scala │ │ │ │ ├── relatedTweetHandlers/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── ConsumersBasedRelatedTweetsHandler.scala │ │ │ │ │ ├── ProducerBasedRelatedTweetsHandler.scala │ │ │ │ │ └── TweetBasedRelatedTweetsHandler.scala │ │ │ │ ├── store/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── UserRecentFollowersStore.scala │ │ │ │ └── util/ │ │ │ │ ├── BUILD │ │ │ │ ├── FetchRHSTweetsUtil.scala │ │ │ │ ├── FilterUtil.scala │ │ │ │ ├── GetAllInternalTweetIdsUtil.scala │ │ │ │ ├── GetRelatedTweetCandidatesUtil.scala │ │ │ │ ├── SampleLHSUsersUtil.scala │ │ │ │ └── UserTweetEdgeTypeMask.scala │ │ │ ├── user_user_graph/ │ │ │ │ ├── BUILD │ │ │ │ ├── KafkaConfig.scala │ │ │ │ ├── LoggingUserUserGraph.scala │ │ │ │ ├── Main.scala │ │ │ │ ├── README.md │ │ │ │ ├── RecommendUsersHandler.scala │ │ │ │ ├── RecosConfig.scala │ │ │ │ ├── UserEdgeTypeMask.scala │ │ │ │ ├── UserUserGraph.scala │ │ │ │ └── UserUserGraphWriter.scala │ │ │ └── user_video_graph/ │ │ │ ├── BUILD │ │ │ ├── LoggingUserVideoGraph.scala │ │ │ ├── Main.scala │ │ │ ├── README.md │ │ │ ├── UserVideoEdgeTypeMask.scala │ │ │ ├── UserVideoGraph.scala │ │ │ ├── UserVideoGraphConfig.scala │ │ │ ├── UserVideoGraphEdgeHttpHandler.scala │ │ │ ├── UserVideoGraphWriter.scala │ │ │ ├── relatedTweetHandlers/ │ │ │ │ ├── BUILD │ │ │ │ ├── ConsumersBasedRelatedTweetsHandler.scala │ │ │ │ ├── ProducerBasedRelatedTweetsHandler.scala │ │ │ │ └── TweetBasedRelatedTweetsHandler.scala │ │ │ ├── store/ │ │ │ │ ├── BUILD │ │ │ │ └── UserRecentFollowersStore.scala │ │ │ └── util/ │ │ │ ├── BUILD │ │ │ ├── FetchRHSTweetsUtil.scala │ │ │ ├── FilterUtil.scala │ │ │ ├── GetAllInternalTweetIdsUtil.scala │ │ │ ├── GetRelatedTweetCandidatesUtil.scala │ │ │ └── SampleLHSUsersUtil.scala │ │ ├── simclusters_v2/ │ │ │ ├── README.md │ │ │ ├── candidate_source/ │ │ │ │ ├── BUILD │ │ │ │ ├── ClusterRanker.scala │ │ │ │ ├── HeavyRanker.scala │ │ │ │ ├── SimClustersANNCandidateSource.scala │ │ │ │ └── SimClustersANNWrapperCandidateSource.scala │ │ │ ├── common/ │ │ │ │ ├── BUILD │ │ │ │ ├── CosineSimilarityUtil.scala │ │ │ │ ├── DeciderGateBuilderWithIdHashing.scala │ │ │ │ ├── ModelVersions.scala │ │ │ │ ├── SeqStandardDeviation.scala │ │ │ │ ├── SimClustersEmbedding.scala │ │ │ │ ├── SimClustersEmbeddingId.scala │ │ │ │ ├── SimClustersEmbeddingIdCacheKeyBuilder.scala │ │ │ │ ├── SimClustersEmbeddingMonoid.scala │ │ │ │ ├── SimClustersMultiEmbedding.scala │ │ │ │ ├── SimClustersMultiEmbeddingId.scala │ │ │ │ ├── clustering/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── ClusterRepresentativeSelectionMethod.scala │ │ │ │ │ ├── ClusteringMethod.scala │ │ │ │ │ ├── ConnectedComponentsClusteringMethod.scala │ │ │ │ │ ├── LargestDimensionClusteringMethod.scala │ │ │ │ │ ├── LouvainClusteringMethod.scala │ │ │ │ │ ├── MaxFavScoreRepresentativeSelectionMethod.scala │ │ │ │ │ ├── MedoidRepresentativeSelectionMethod.scala │ │ │ │ │ └── SimilarityFunctions.scala │ │ │ │ ├── ml/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── SimClustersEmbeddingAdapter.scala │ │ │ │ └── package.scala │ │ │ ├── hdfs_sources/ │ │ │ │ ├── AdhocSources.scala │ │ │ │ ├── BUILD │ │ │ │ ├── DataPaths.scala │ │ │ │ ├── DataSources.scala │ │ │ │ ├── EntityEmbeddingsSources.scala │ │ │ │ ├── InterestedInSources.scala │ │ │ │ ├── ProducerEmbeddingSources.scala │ │ │ │ ├── injections/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── ClusterDetailsInjection.scala │ │ │ │ │ ├── ClusterTopMediaTweetsInjection.scala │ │ │ │ │ ├── ClusterTopTweetsInjection.scala │ │ │ │ │ ├── ClusteringInjections.scala │ │ │ │ │ ├── EntityEmbeddingsInjections.scala │ │ │ │ │ ├── InferredEntitiesInjections.scala │ │ │ │ │ ├── InterestedInInjection.scala │ │ │ │ │ ├── KnownForInjection.scala │ │ │ │ │ ├── MultiTypeGraphInjections.scala │ │ │ │ │ ├── ProducerEmbeddingsInjections.scala │ │ │ │ │ ├── SemanticCoreEntitiesInjections.scala │ │ │ │ │ └── SingleSideUserScoresInjection.scala │ │ │ │ └── presto_hdfs_sources/ │ │ │ │ ├── BUILD │ │ │ │ └── EntityEmbeddingsPrestoSources.scala │ │ │ ├── scalding/ │ │ │ │ ├── BUILD │ │ │ │ ├── BipartiteClusterEvaluation.scala │ │ │ │ ├── BipartiteClusterEvaluationClasses.scala │ │ │ │ ├── ClusterDetailsJob.scala │ │ │ │ ├── ClusterEvaluation.scala │ │ │ │ ├── CompareClusters.scala │ │ │ │ ├── EigenVectorsForSparseSymmetric.scala │ │ │ │ ├── InterestedInFromAggregatableProducerEmbeddings.scala │ │ │ │ ├── InterestedInFromKnownFor.scala │ │ │ │ ├── InterestedInFromKnownForLite.scala │ │ │ │ ├── InterestedInFromProducerEmbeddingsAdhocApp.scala │ │ │ │ ├── KnownForSources.scala │ │ │ │ ├── ProducerNormsAndCounts.scala │ │ │ │ ├── TopUsersSimilarityGraph.scala │ │ │ │ ├── UpdateKnownFor.scala │ │ │ │ ├── UpdateKnownForApps.scala │ │ │ │ ├── UserUserFavGraph.scala │ │ │ │ ├── UserUserGraph.scala │ │ │ │ ├── UserUserNormalizedGraph.scala │ │ │ │ ├── common/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── PersistentTweetEmbeddingSource.scala │ │ │ │ │ ├── QTreeMultiAggregator.scala │ │ │ │ │ ├── TypedRichPipe.scala │ │ │ │ │ ├── Util.scala │ │ │ │ │ └── matrix/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── DenseRowMatrix.scala │ │ │ │ │ ├── SparseMatrix.scala │ │ │ │ │ ├── SparseRowMatrix.scala │ │ │ │ │ └── TypedPipeMatrix.scala │ │ │ │ ├── embedding/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── EntityEmbeddingFromProducerEmbeddingJob.scala │ │ │ │ │ ├── EntityToSimClustersEmbeddingsJob.scala │ │ │ │ │ ├── GlobalSimClustersLanguageEmbedding.scala │ │ │ │ │ ├── LocaleEntitySimClustersEmbeddingV2Job.scala │ │ │ │ │ ├── LocaleEntitySimClustersEmbeddingsJob.scala │ │ │ │ │ ├── ProducerEmbeddingsFromInterestedIn.scala │ │ │ │ │ ├── SimilarUsersBySimClustersEmbedding.scala │ │ │ │ │ ├── abuse/ │ │ │ │ │ │ ├── AbuseSimclusterFeaturesScaldingJob.scala │ │ │ │ │ │ ├── AdhocAbuseSimClusterFeaturesScaldingJob.scala │ │ │ │ │ │ ├── BUILD │ │ │ │ │ │ ├── CrossSimClusterFeaturesScaldingJob.scala │ │ │ │ │ │ ├── DataSources.scala │ │ │ │ │ │ ├── PairedinteractionFeatures.scala │ │ │ │ │ │ └── SingleSideInteractionTransformation.scala │ │ │ │ │ ├── common/ │ │ │ │ │ │ ├── EmbeddingUtil.scala │ │ │ │ │ │ ├── EntityEmbeddingUtil.scala │ │ │ │ │ │ ├── ExternalDataSources.scala │ │ │ │ │ │ └── SimClustersEmbeddingJob.scala │ │ │ │ │ ├── producer/ │ │ │ │ │ │ ├── AggregatableFavBasedProducerEmbeddings.scala │ │ │ │ │ │ ├── AggregatableFollowBasedProducerEmbeddings.scala │ │ │ │ │ │ ├── AggregatableLogFavBasedProducerEmbeddings.scala │ │ │ │ │ │ ├── AggregatableProducerEmbeddings.scala │ │ │ │ │ │ └── BUILD.bazel │ │ │ │ │ ├── tfg/ │ │ │ │ │ │ ├── BUILD │ │ │ │ │ │ ├── EngagementWeightedTfgBasedTopicEmbeddingsJob.scala │ │ │ │ │ │ ├── FavInferredLanguageTfgBasedTopicEmbeddings.scala │ │ │ │ │ │ ├── FavTfgBasedTopicEmbeddings.scala │ │ │ │ │ │ ├── InferredLanguageTfgBasedTopicEmbeddingsBaseApp.scala │ │ │ │ │ │ ├── LogFavTfgBasedTopicEmbeddings.scala │ │ │ │ │ │ ├── README │ │ │ │ │ │ └── TfgBasedTopicEmbeddingsBaseApp.scala │ │ │ │ │ └── twice/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ ├── InterestedInTwice.scala │ │ │ │ │ └── InterestedInTwiceBaseApp.scala │ │ │ │ ├── evaluation/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ ├── CandidateEvaluationBase.scala │ │ │ │ │ ├── EvaluationMetricHelper.scala │ │ │ │ │ ├── EvaluationReferenceDataExtraction.scala │ │ │ │ │ ├── LabelCorrelationsHelper.scala │ │ │ │ │ └── SimClustersEvaluationAdhocApp.scala │ │ │ │ ├── inferred_entities/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ ├── InferredEntities.scala │ │ │ │ │ ├── InferredEntitiesFromInterestedIn.scala │ │ │ │ │ ├── InferredSemanticCoreEntitiesFromKnownFor.scala │ │ │ │ │ └── ProdSources.scala │ │ │ │ ├── mbcg/ │ │ │ │ │ ├── AllFeatures.scala │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ ├── RecordAdapters.scala │ │ │ │ │ ├── TweetEmbeddingGenerationJob.scala │ │ │ │ │ └── UserEmbeddingGenerationJob.scala │ │ │ │ ├── multi_type_graph/ │ │ │ │ │ └── assemble_multi_type_graph/ │ │ │ │ │ ├── AssembleMultiTypeGraph.scala │ │ │ │ │ ├── AssembleMultiTypeGraphApp.scala │ │ │ │ │ ├── AssembleMultiTypeGraphBaseApp.scala │ │ │ │ │ ├── BUILD │ │ │ │ │ └── Config.scala │ │ │ │ ├── offline_job/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ ├── OfflineTweetRecommendation.scala │ │ │ │ │ ├── SimClustersOfflineJob.scala │ │ │ │ │ ├── SimClustersOfflineJobAdhocApp.scala │ │ │ │ │ ├── SimClustersOfflineJobScheduledApp.scala │ │ │ │ │ ├── SimClustersOfflineJobUtil.scala │ │ │ │ │ └── adhoc/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ ├── README │ │ │ │ │ ├── SimClustersTweetEmbeddingAdhocApp.scala │ │ │ │ │ └── TweetSimilarityEvaluationAdhocApp.scala │ │ │ │ ├── offline_tweets/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ └── ClusterTopMediaTweetsJob.scala │ │ │ │ ├── optout/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ ├── InterestedInOptOut.scala │ │ │ │ │ ├── KnownForOptOut.scala │ │ │ │ │ └── SimClustersOptOutUtil.scala │ │ │ │ ├── topic_recommendations/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── GeoPopularTopicsApp.scala │ │ │ │ │ ├── ProducersForTopicsFromTopicFollowGraph.scala │ │ │ │ │ ├── SimilarTopicsFromTopicFollowGraphApp.scala │ │ │ │ │ ├── TopicsForProducersFromEM.scala │ │ │ │ │ ├── TopicsForProducersUtils.scala │ │ │ │ │ └── model_based_topic_recommendations/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── DataSources.scala │ │ │ │ │ ├── UserFeatures.scala │ │ │ │ │ ├── UserTopicDataRecordAdapter.scala │ │ │ │ │ └── UserTopicModellingTrainingDataCollectionJob.scala │ │ │ │ ├── tweet_similarity/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── DatasetTopKAnalysisJob.scala │ │ │ │ │ ├── TrainingDataCollectionJob.scala │ │ │ │ │ ├── TrainingDataCollectionUtil.scala │ │ │ │ │ ├── TweetPairFeatureHydrationUtil.scala │ │ │ │ │ ├── TweetPairLabelCollectionUtil.scala │ │ │ │ │ ├── UnhydratedPairsCollectionJob.scala │ │ │ │ │ └── evaluation/ │ │ │ │ │ ├── BUILD.bazel │ │ │ │ │ ├── ModelEvalAdhocApp.scala │ │ │ │ │ └── RUXLandingDdgAnalysisAdhocApp.scala │ │ │ │ └── update_known_for/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── UpdateKnownFor20M145K2020.scala │ │ │ │ └── UpdateKnownForSBFRunner.scala │ │ │ ├── scio/ │ │ │ │ ├── bq_generation/ │ │ │ │ │ ├── common/ │ │ │ │ │ │ ├── BQGenerationUtil.scala │ │ │ │ │ │ ├── BUILD │ │ │ │ │ │ └── IndexGenerationUtil.scala │ │ │ │ │ ├── ftr_tweet/ │ │ │ │ │ │ ├── BUILD │ │ │ │ │ │ ├── Config.scala │ │ │ │ │ │ ├── FTRJob.scala │ │ │ │ │ │ ├── FtrClusterToTweetIndexGenerationJob.scala │ │ │ │ │ │ ├── README.md │ │ │ │ │ │ ├── ftr-based-simclusters-index-generation-job.d6w │ │ │ │ │ │ ├── ftr-tweets-ann-adhoc-job.d6w │ │ │ │ │ │ ├── iikf2020-decayed-sum-ann-batch-job.d6w │ │ │ │ │ │ ├── iikf2020-ftrat5-pop1000-ann-batch-job.d6w │ │ │ │ │ │ ├── iikf2020-ftrat5-pop10000-ann-batch-job.d6w │ │ │ │ │ │ └── sql/ │ │ │ │ │ │ ├── BUILD │ │ │ │ │ │ └── ftr_tweet_embeddings.sql │ │ │ │ │ ├── simclusters_index_generation/ │ │ │ │ │ │ ├── BUILD │ │ │ │ │ │ ├── Config.scala │ │ │ │ │ │ ├── EngagementEventBasedClusterToTweetIndexFromBQ.scala │ │ │ │ │ │ ├── EngagementEventBasedClusterToTweetIndexGenerationJob.scala │ │ │ │ │ │ ├── README │ │ │ │ │ │ └── engagement-event-based-simclusters-index-generation-job.d6w │ │ │ │ │ ├── sql/ │ │ │ │ │ │ ├── BUILD │ │ │ │ │ │ ├── ads_user_tweet_action_pair_generation.sql │ │ │ │ │ │ ├── cluster_top_tweets.sql │ │ │ │ │ │ ├── cluster_top_tweets_intersection_with_fav_based_index.sql │ │ │ │ │ │ ├── combined_user_tweet_action_pair_generation.sql │ │ │ │ │ │ ├── engagement_based_index_generation.sql │ │ │ │ │ │ ├── evergreen_content_user_tweet_action_pair_generation.sql │ │ │ │ │ │ ├── nsfw_tweet_denylist.sql │ │ │ │ │ │ ├── tweet_embeddings_generation.sql │ │ │ │ │ │ ├── tweet_fav_count.sql │ │ │ │ │ │ ├── tweets_ann.sql │ │ │ │ │ │ ├── unified_user_tweet_action_pair_generation.sql │ │ │ │ │ │ └── user_video_tweet_fav_engagement_generation.sql │ │ │ │ │ └── tweets_ann/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── Config.scala │ │ │ │ │ ├── README │ │ │ │ │ ├── TweetsANNFromBQ.scala │ │ │ │ │ ├── TweetsANNJob.scala │ │ │ │ │ ├── iikf-hl-0-el-15-tweets-ann-batch-job.d6w │ │ │ │ │ ├── iikf-hl-2-el-15-tweets-ann-batch-job.d6w │ │ │ │ │ ├── iikf-hl-2-el-50-tweets-ann-batch-job.d6w │ │ │ │ │ ├── iikf-hl-8-el-50-tweets-ann-adhoc-job.d6w │ │ │ │ │ ├── iikf-hl-8-el-50-tweets-ann-batch-job.d6w │ │ │ │ │ ├── iikf-tweets-ann-adhoc-job.d6w │ │ │ │ │ ├── iikf-tweets-ann-batch-job.d6w │ │ │ │ │ ├── mts-consumer-embeddings-tweets-ann-adhoc-job.d6w │ │ │ │ │ └── mts-consumer-embeddings-tweets-ann-batch-job.d6w │ │ │ │ ├── common/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── ExternalDataSources.scala │ │ │ │ └── multi_type_graph/ │ │ │ │ ├── assemble_multi_type_graph/ │ │ │ │ │ ├── AssembleMultiTypeGraphScioApp.scala │ │ │ │ │ ├── AssembleMultiTypeGraphScioBaseApp.scala │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── Config.scala │ │ │ │ │ ├── README.md │ │ │ │ │ ├── assemble-multi-type-graph-scio-adhoc.d6w │ │ │ │ │ └── assemble-multi-type-graph-scio-batch.d6w │ │ │ │ ├── common/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── MultiTypeGraphUtil.scala │ │ │ │ └── multi_type_graph_sims/ │ │ │ │ ├── BUILD │ │ │ │ ├── Config.scala │ │ │ │ ├── RightNodeCosineSimilarityScioApp.scala │ │ │ │ ├── RightNodeCosineSimilarityScioBaseApp.scala │ │ │ │ ├── RightNodeSimHashScioApp.scala │ │ │ │ ├── RightNodeSimHashScioBaseApp.scala │ │ │ │ ├── cosine-similarity-scio-adhoc.d6w │ │ │ │ ├── cosine-similarity-scio-batch.d6w │ │ │ │ ├── sim-hash-scio-adhoc.d6w │ │ │ │ └── sim-hash-scio-batch.d6w │ │ │ ├── score/ │ │ │ │ ├── AggregatedScoreStore.scala │ │ │ │ ├── BUILD │ │ │ │ ├── Score.scala │ │ │ │ ├── ScoreFacadeStore.scala │ │ │ │ ├── ScoreId.scala │ │ │ │ ├── ScoreStore.scala │ │ │ │ ├── SimClustersEmbeddingPairScoreStore.scala │ │ │ │ └── WeightedSumAggregatedScoreStore.scala │ │ │ ├── stores/ │ │ │ │ ├── BUILD │ │ │ │ ├── LanguageFilteredLocaleEntityEmbeddingStore.scala │ │ │ │ ├── MultiTypeGraphStore.scala │ │ │ │ ├── SimClustersEmbeddingStore.scala │ │ │ │ ├── SimClustersMultiEmbeddingStore.scala │ │ │ │ ├── TopicTopProducersStore.scala │ │ │ │ └── WtfMbcgStore.scala │ │ │ ├── summingbird/ │ │ │ │ ├── BUILD │ │ │ │ ├── README.md │ │ │ │ ├── common/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── ClientConfigs.scala │ │ │ │ │ ├── Configs.scala │ │ │ │ │ ├── EntityUtil.scala │ │ │ │ │ ├── Implicits.scala │ │ │ │ │ ├── ModelVersionProfile.scala │ │ │ │ │ ├── Monoids.scala │ │ │ │ │ ├── SimClustersEmbeddingWithMetadataMonoid.scala │ │ │ │ │ ├── SimClustersHashUtil.scala │ │ │ │ │ ├── SimClustersInterestedInUtil.scala │ │ │ │ │ ├── SimClustersProfile.scala │ │ │ │ │ ├── StatsUtil.scala │ │ │ │ │ ├── SummerWithSumValues.scala │ │ │ │ │ ├── ThriftDecayedValueMonoid.scala │ │ │ │ │ └── TweetEntityExtractor.scala │ │ │ │ ├── stores/ │ │ │ │ │ ├── ApeTopicEmbeddingStore.scala │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── ClusterDetailsReadableStore.scala │ │ │ │ │ ├── EntityClusterScoreReadableStore.scala │ │ │ │ │ ├── ManhattanFromStratoStore.scala │ │ │ │ │ ├── PersistentTweetEmbeddingStore.scala │ │ │ │ │ ├── ProducerClusterEmbeddingReadableStores.scala │ │ │ │ │ ├── SemanticCoreEntityEmbeddingStore.scala │ │ │ │ │ ├── SimClustersManhattanReadableStoreForReadWriteDataset.scala │ │ │ │ │ ├── TfgTopicEmbeddingsStore.scala │ │ │ │ │ ├── TopKClustersForEntityReadableStore.scala │ │ │ │ │ ├── TopKClustersForTweetReadableStore.scala │ │ │ │ │ ├── TopKTweetsForClusterReadableStore.scala │ │ │ │ │ ├── TweetStatusCountsStore.scala │ │ │ │ │ ├── UserInterestedInReadableStore.scala │ │ │ │ │ └── UserKnownForReadableStore.scala │ │ │ │ └── storm/ │ │ │ │ ├── BUILD │ │ │ │ ├── PersistentTweetJob.scala │ │ │ │ ├── PersistentTweetJobRunner.scala │ │ │ │ ├── TweetJob.scala │ │ │ │ ├── TweetJobRunner.scala │ │ │ │ ├── persistent_tweet_job_deploy.sh │ │ │ │ ├── tweet_alt_job_deploy.sh │ │ │ │ └── tweet_job_deploy.sh │ │ │ └── tweet_similarity/ │ │ │ ├── BUILD │ │ │ ├── ModelBasedTweetSimilaritySimClustersEmbeddingAdapter.scala │ │ │ └── TweetSimilarityFeatures.scala │ │ └── timelines/ │ │ └── prediction/ │ │ ├── common/ │ │ │ └── aggregates/ │ │ │ ├── BCELabelTransformFromUUADataRecord.scala │ │ │ ├── BUILD │ │ │ ├── FeatureSelectorConfig.scala │ │ │ ├── README.md │ │ │ ├── RecapUserFeatureAggregation.scala │ │ │ ├── RectweetUserFeatureAggregation.scala │ │ │ ├── TimelinesAggregationConfig.scala │ │ │ ├── TimelinesAggregationConfigDetails.scala │ │ │ ├── TimelinesAggregationConfigTrait.scala │ │ │ ├── TimelinesAggregationKeyValInjections.scala │ │ │ ├── TimelinesAggregationSources.scala │ │ │ └── real_time/ │ │ │ ├── AuthorFeaturesAdapter.scala │ │ │ ├── BUILD │ │ │ ├── Event.scala │ │ │ ├── FeatureStoreUtils.scala │ │ │ ├── LocallyReplicatedStore.scala │ │ │ ├── StormAggregateSourceUtils.scala │ │ │ ├── TimelinesOnlineAggregationConfig.scala │ │ │ ├── TimelinesOnlineAggregationConfigBase.scala │ │ │ ├── TimelinesOnlineAggregationSources.scala │ │ │ ├── TimelinesRealTimeAggregatesJob.scala │ │ │ ├── TimelinesStormAggregateSource.scala │ │ │ ├── TweetFeaturesAdapter.scala │ │ │ ├── TweetFeaturesReadableStore.scala │ │ │ ├── TypeSafeRunner.scala │ │ │ ├── UserFeaturesAdapter.scala │ │ │ └── UserFeaturesReadableStore.scala │ │ └── features/ │ │ ├── README.md │ │ ├── client_log_event/ │ │ │ ├── BUILD │ │ │ └── ClientLogEventDataRecordFeatures.scala │ │ ├── common/ │ │ │ ├── BUILD │ │ │ ├── CombinedFeatures.scala │ │ │ ├── NonHomeLabelFeatures.scala │ │ │ └── TimelinesSharedFeatures.scala │ │ ├── engagement_features/ │ │ │ ├── BUILD │ │ │ └── EngagementFeatures.scala │ │ ├── escherbird/ │ │ │ ├── BUILD │ │ │ ├── EscherbirdFeatures.scala │ │ │ └── EscherbirdFeaturesConverter.scala │ │ ├── followsource/ │ │ │ ├── BUILD.bazel │ │ │ └── FollowSourceFeatures.scala │ │ ├── itl/ │ │ │ ├── BUILD │ │ │ └── ITLFeatures.scala │ │ ├── list_features/ │ │ │ ├── BUILD │ │ │ └── ListFeatures.scala │ │ ├── p_home_latest/ │ │ │ ├── BUILD │ │ │ └── HomeLatestUserFeatures.scala │ │ ├── ppmi/ │ │ │ ├── BUILD │ │ │ └── PpmiFeatures.scala │ │ ├── real_graph/ │ │ │ ├── BUILD │ │ │ ├── RealGraphDataRecordFeatureStoreFeatures.scala │ │ │ └── RealGraphDataRecordFeatures.scala │ │ ├── recap/ │ │ │ ├── BUILD │ │ │ ├── RecapFeatures.scala │ │ │ └── RecapFeaturesUtils.scala │ │ ├── request_context/ │ │ │ ├── BUILD │ │ │ └── RequestContextFeatures.scala │ │ ├── simcluster/ │ │ │ ├── BUILD │ │ │ ├── SimclusterFeatures.scala │ │ │ ├── SimclusterTweetFeatures.scala │ │ │ └── SimclustersScoresFeatures.scala │ │ ├── socialproof/ │ │ │ ├── BUILD │ │ │ └── SocialProofFeatures.scala │ │ ├── time_features/ │ │ │ ├── BUILD │ │ │ └── TimeDataRecordFeatures.scala │ │ ├── two_hop_features/ │ │ │ ├── BUILD │ │ │ ├── TwoHopFeatures.scala │ │ │ └── TwoHopFeaturesConfig.scala │ │ └── user_health/ │ │ ├── BUILD │ │ └── UserHealthFeatures.scala │ └── thrift/ │ └── com/ │ └── twitter/ │ ├── interaction_graph/ │ │ ├── BUILD │ │ └── interaction_graph.thrift │ ├── recos/ │ │ ├── recos.thrift │ │ ├── recos_common.thrift │ │ ├── recos_injector.thrift │ │ ├── user_tweet_entity_graph/ │ │ │ ├── BUILD │ │ │ ├── CONFIG.ini │ │ │ └── user_tweet_entity_graph.thrift │ │ ├── user_tweet_graph/ │ │ │ ├── BUILD │ │ │ ├── CONFIG.ini │ │ │ └── user_tweet_graph.thrift │ │ ├── user_user_graph/ │ │ │ ├── BUILD │ │ │ ├── CONFIG.ini │ │ │ └── user_user_graph.thrift │ │ └── user_video_graph/ │ │ ├── BUILD │ │ ├── CONFIG.ini │ │ └── user_video_graph.thrift │ ├── search/ │ │ ├── common/ │ │ │ └── ranking/ │ │ │ └── ranking.thrift │ │ └── earlybird/ │ │ └── thrift/ │ │ └── earlybird.thrift │ └── simclusters_v2/ │ ├── BUILD │ ├── abuse.thrift │ ├── clustering.thrift │ ├── embedding.thrift │ ├── entity.thrift │ ├── evaluation.thrift │ ├── graph.thrift │ ├── identifier.thrift │ ├── inferred_entities.thrift │ ├── interests.thrift │ ├── multi_type_graph.thrift │ ├── offline_job_internal.thrift │ ├── online_store.thrift │ ├── online_store_internal.thrift │ ├── score.thrift │ ├── simclusters_presto.thrift │ ├── top_k_map.thrift │ └── tweet_similarity.thrift ├── timelineranker/ │ ├── README.md │ ├── client/ │ │ └── builder/ │ │ ├── BUILD │ │ ├── README.md │ │ └── src/ │ │ └── main/ │ │ └── scala/ │ │ ├── BUILD │ │ └── com/ │ │ └── twitter/ │ │ └── timelineranker/ │ │ └── client/ │ │ ├── TimelineRankerClient.scala │ │ └── TimelineRankerClientBuilder.scala │ ├── common/ │ │ ├── BUILD │ │ └── src/ │ │ └── main/ │ │ └── scala/ │ │ ├── BUILD │ │ └── com/ │ │ └── twitter/ │ │ └── timelineranker/ │ │ ├── adapter/ │ │ │ ├── BUILD │ │ │ └── TimelineServiceAdapter.scala │ │ └── model/ │ │ ├── BUILD │ │ ├── CandidateTweet.scala │ │ ├── CandidateTweetsResult.scala │ │ ├── HydratedTweetEntry.scala │ │ ├── Language.scala │ │ ├── LanguageScope.scala │ │ ├── PartiallyHydratedTweet.scala │ │ ├── PriorSeenEntries.scala │ │ ├── RankedTimelineQuery.scala │ │ ├── RankedTimelineQueryOptions.scala │ │ ├── RecapQuery.scala │ │ ├── ReverseChronTimelineQuery.scala │ │ ├── ReverseChronTimelineQueryOptions.scala │ │ ├── TimeRange.scala │ │ ├── Timeline.scala │ │ ├── TimelineEntry.scala │ │ ├── TimelineEntryEnvelope.scala │ │ ├── TimelineQuery.scala │ │ ├── TimelineQueryOptions.scala │ │ ├── TimelineRange.scala │ │ ├── Tweet.scala │ │ ├── TweetIdRange.scala │ │ └── UtegLikedByTweetsOptions.scala │ └── server/ │ ├── BUILD.bazel │ ├── config/ │ │ ├── BUILD │ │ └── decider.yml │ └── src/ │ └── main/ │ ├── resources/ │ │ ├── BUILD.bazel │ │ └── logback-timelineranker.xml │ └── scala/ │ ├── BUILD.bazel │ └── com/ │ └── twitter/ │ └── timelineranker/ │ ├── clients/ │ │ ├── BUILD │ │ ├── CortexTweetQueryServiceClient.scala │ │ ├── MemcacheFactory.scala │ │ └── content_features_cache/ │ │ ├── BUILD │ │ └── ContentFeaturesMemcacheBuilder.scala │ ├── common/ │ │ ├── BUILD │ │ ├── CandidateGenerationTransform.scala │ │ ├── ContentFeaturesHydrationTransform.scala │ │ ├── CreateCandidateEnvelopeTransform.scala │ │ ├── FeatureHydrationDataTransform.scala │ │ ├── FollowAndRealGraphCombiningTransform.scala │ │ ├── FollowGraphDataTransform.scala │ │ ├── HydrateTweetsAndSourceTweetsInParallelTransform.scala │ │ ├── HydratedTweetsFilterTransform.scala │ │ ├── InNetworkTweetsSearchFeaturesHydrationTransform.scala │ │ ├── MarkRandomTweetTransform.scala │ │ ├── OutOfNetworkRepliesToUserIdSearchResultsTransform.scala │ │ ├── OutOfNetworkTweetsSearchFeaturesHydrationTransform.scala │ │ ├── RecapHydrationSearchResultsTransformBase.scala │ │ ├── RecapSearchResultsTransform.scala │ │ ├── RecapSearchResultsTruncationTransform.scala │ │ ├── SearchResultDedupAndSortingTransform.scala │ │ ├── SourceTweetsSearchResultsTransform.scala │ │ ├── TrimToMatchHydratedTweetsTransform.scala │ │ ├── TrimToMatchSearchResultsTransform.scala │ │ ├── TweetHydrationTransform.scala │ │ ├── TweetKindOptionHydratedTweetsFilterTransform.scala │ │ ├── UserLanguagesTransform.scala │ │ ├── UserProfileInfoTransform.scala │ │ └── VisibilityEnforcingTransform.scala │ ├── config/ │ │ ├── BUILD │ │ ├── CallInfo.scala │ │ ├── ClientAccessPermissions.scala │ │ ├── ClientWrapperFactories.scala │ │ ├── ClientWrappers.scala │ │ ├── DefaultUnderlyingClientConfiguration.scala │ │ ├── RequestScopes.scala │ │ ├── RuntimeConfiguration.scala │ │ ├── StagingUnderlyingConfiguration.scala │ │ ├── TimelineRankerConstants.scala │ │ ├── TimelineRankerFlags.scala │ │ └── UnderlyingClientConfiguration.scala │ ├── contentfeatures/ │ │ ├── BUILD │ │ └── package.scala │ ├── core/ │ │ ├── BUILD │ │ ├── CandidateEnvelope.scala │ │ ├── FollowGraphData.scala │ │ ├── FollowGraphDataFuture.scala │ │ ├── HydratedCandidatesAndFeaturesEnvelope.scala │ │ ├── HydratedTweets.scala │ │ └── package.scala │ ├── decider/ │ │ ├── BUILD │ │ └── DeciderKey.scala │ ├── entity_tweets/ │ │ ├── BUILD.bazel │ │ ├── EntityTweetsRepository.scala │ │ ├── EntityTweetsRepositoryBuilder.scala │ │ ├── EntityTweetsSearchResultsTransform.scala │ │ └── EntityTweetsSource.scala │ ├── in_network_tweets/ │ │ ├── BUILD │ │ ├── InNetworkTweetRepository.scala │ │ ├── InNetworkTweetRepositoryBuilder.scala │ │ └── InNetworkTweetSource.scala │ ├── monitoring/ │ │ ├── BUILD │ │ └── UsersSearchResultMonitoringTransform.scala │ ├── observe/ │ │ ├── BUILD.bazel │ │ ├── DebugObserverBuilder.scala │ │ └── ObservedRequests.scala │ ├── parameters/ │ │ ├── BUILD │ │ ├── ConfigBuilder.scala │ │ ├── entity_tweets/ │ │ │ ├── BUILD │ │ │ ├── EntityTweetsParams.scala │ │ │ └── EntityTweetsProduction.scala │ │ ├── in_network_tweets/ │ │ │ ├── BUILD │ │ │ ├── InNetworkTweetParams.scala │ │ │ └── InNetworkTweetProduction.scala │ │ ├── monitoring/ │ │ │ ├── BUILD │ │ │ ├── MonitoringParams.scala │ │ │ └── MonitoringProduction.scala │ │ ├── recap/ │ │ │ ├── BUILD │ │ │ ├── RecapParams.scala │ │ │ ├── RecapProduction.scala │ │ │ └── RecapQueryContext.scala │ │ ├── recap_author/ │ │ │ ├── BUILD │ │ │ ├── RecapAuthorParams.scala │ │ │ └── RecapAuthorProduction.scala │ │ ├── recap_hydration/ │ │ │ ├── BUILD │ │ │ ├── RecapHydrationParams.scala │ │ │ └── RecapHydrationProduction.scala │ │ ├── revchron/ │ │ │ ├── BUILD │ │ │ ├── ReverseChronParams.scala │ │ │ ├── ReverseChronProduction.scala │ │ │ ├── ReverseChronTimelineQueryContext.scala │ │ │ └── ReverseChronTimelineQueryContextBuilder.scala │ │ ├── uteg_liked_by_tweets/ │ │ │ ├── BUILD │ │ │ ├── UtegLikedByTweetsParams.scala │ │ │ └── UtegLikedByTweetsProduction.scala │ │ └── util/ │ │ ├── BUILD │ │ ├── CommonRequestContext.scala │ │ ├── ConfigHelper.scala │ │ └── RecapQueryParamInitializer.scala │ ├── recap/ │ │ ├── BUILD │ │ └── model/ │ │ ├── BUILD │ │ └── ContentFeatures.scala │ ├── recap_author/ │ │ ├── BUILD.bazel │ │ ├── RecapAuthorRepository.scala │ │ ├── RecapAuthorRepositoryBuilder.scala │ │ ├── RecapAuthorSearchResultsTransform.scala │ │ └── RecapAuthorSource.scala │ ├── recap_hydration/ │ │ ├── BUILD.bazel │ │ ├── RecapHydrationRepository.scala │ │ ├── RecapHydrationRepositoryBuilder.scala │ │ ├── RecapHydrationSearchResultsTransform.scala │ │ └── RecapHydrationSource.scala │ ├── repository/ │ │ ├── BUILD │ │ ├── CandidatesRepositoryBuilder.scala │ │ ├── RankedHomeTimelineRepository.scala │ │ ├── RepositoryBuilder.scala │ │ ├── ReverseChronHomeTimelineRepository.scala │ │ ├── ReverseChronHomeTimelineRepositoryBuilder.scala │ │ ├── RoutingTimelineRepository.scala │ │ ├── RoutingTimelineRepositoryBuilder.scala │ │ └── TimelineRepository.scala │ ├── server/ │ │ ├── BUILD.bazel │ │ ├── Main.scala │ │ ├── TimelineRanker.scala │ │ ├── TimelineRankerBuilder.scala │ │ ├── TimelineRankerThriftWebForms.scala │ │ └── Warmup.scala │ ├── source/ │ │ ├── BUILD │ │ ├── ReverseChronHomeTimelineSource.scala │ │ └── TimelineSource.scala │ ├── uteg_liked_by_tweets/ │ │ ├── BUILD.bazel │ │ ├── CombinedScoreAndTruncateTransform.scala │ │ ├── MinNumNonAuthorFavoritedByUserIdsFilterTransform.scala │ │ ├── RemoveCandidatesAuthoredByWeightedFollowingsTransform.scala │ │ ├── SocialProofAndUTEGScoreHydrationTransform.scala │ │ ├── UTEGResultsTransform.scala │ │ ├── UtegLikedByTweetsRepository.scala │ │ ├── UtegLikedByTweetsRepositoryBuilder.scala │ │ ├── UtegLikedByTweetsSearchResultsTransform.scala │ │ └── UtegLikedByTweetsSource.scala │ ├── util/ │ │ ├── BUILD │ │ ├── CachingContentFeaturesProvider.scala │ │ ├── CopyContentFeaturesIntoHydratedTweetsTransform.scala │ │ ├── CopyContentFeaturesIntoThriftTweetFeaturesTransform.scala │ │ ├── ExtendedRepliesFilter.scala │ │ ├── LatentRepository.scala │ │ ├── RecommendedRepliesFilter.scala │ │ ├── ReverseExtendedRepliesFilter.scala │ │ ├── SearchResultUtil.scala │ │ ├── SearchResultWithVisibilityActors.scala │ │ ├── SnowflakeUtils.scala │ │ ├── SourceTweetsUtil.scala │ │ ├── TweetAnnotationFeaturesExtractor.scala │ │ ├── TweetHydrator.scala │ │ ├── TweetMediaFeatureExtractor.scala │ │ ├── TweetTextFeaturesExtractor.scala │ │ ├── TweetsPostFilter.scala │ │ ├── TweetsPostFilterBasedOnSearchMetadata.scala │ │ └── TweetypieContentFeaturesProvider.scala │ └── visibility/ │ ├── BUILD │ ├── FollowGraphDataProvider.scala │ ├── RealGraphFollowGraphDataProvider.scala │ └── SgsFollowGraphDataProvider.scala ├── timelines/ │ └── data_processing/ │ ├── ad_hoc/ │ │ └── earlybird_ranking/ │ │ └── earlybird_ranking/ │ │ ├── BUILD │ │ ├── common/ │ │ │ ├── BUILD │ │ │ ├── EarlybirdTrainingConfiguration.scala │ │ │ ├── EarlybirdTrainingRecapConfiguration.scala │ │ │ └── EarlybirdTrainingRectweetConfiguration.scala │ │ ├── model_evaluation/ │ │ │ ├── BUILD │ │ │ ├── EarlybirdEvaluationMetric.scala │ │ │ └── EarlybirdModelEvaluationJob.scala │ │ └── training_data_generation/ │ │ ├── BUILD │ │ ├── EarlybirdExampleSampler.scala │ │ ├── EarlybirdStatsJob.scala │ │ └── EarlybirdTrainingDataJob.scala │ └── ml_util/ │ └── aggregation_framework/ │ ├── AggregateGroup.scala │ ├── AggregateSource.scala │ ├── AggregateStore.scala │ ├── AggregationConfig.scala │ ├── AggregationKey.scala │ ├── BUILD │ ├── DataRecordAggregationMonoid.scala │ ├── KeyedRecord.scala │ ├── OfflineAggregateInjections.scala │ ├── OfflineAggregateSource.scala │ ├── OfflineAggregateStore.scala │ ├── README.md │ ├── StoreConfig.scala │ ├── StoreRegister.scala │ ├── TypedAggregateGroup.scala │ ├── Utils.scala │ ├── conversion/ │ │ ├── AggregatesV2Adapter.scala │ │ ├── AggregatesV2FeatureSource.scala │ │ ├── BUILD │ │ ├── CombineCountsPolicy.scala │ │ ├── DataSetPipeSketchJoin.scala │ │ ├── PickFirstRecordPolicy.scala │ │ ├── PickTopCtrPolicy.scala │ │ ├── SparseBinaryAggregateJoin.scala │ │ ├── SparseBinaryMergePolicy.scala │ │ └── SparseBinaryMultipleAggregateJoin.scala │ ├── docs/ │ │ ├── AUTOMATED_COMMIT_FILES │ │ ├── aggregation.rst │ │ ├── batch.rst │ │ ├── conf.py │ │ ├── index.rst │ │ ├── joining.rst │ │ ├── real-time.rst │ │ └── troubleshooting.rst │ ├── heron/ │ │ ├── BUILD │ │ ├── NighthawkUnderlyingStoreConfig.scala │ │ ├── OnlineAggregationConfigTrait.scala │ │ ├── OnlineAggregationStoresTrait.scala │ │ ├── RealTimeAggregateStore.scala │ │ ├── RealTimeAggregatesJobBase.scala │ │ ├── RealTimeAggregatesJobConfig.scala │ │ ├── StormAggregateSource.scala │ │ ├── UserReindexingNighthawkStore.scala │ │ └── package.scala │ ├── job/ │ │ ├── AggregatesV2Job.scala │ │ ├── BUILD │ │ └── DataRecordFeatureCounter.scala │ ├── metrics/ │ │ ├── AggregateFeature.scala │ │ ├── AggregationMetric.scala │ │ ├── AggregationMetricCommon.scala │ │ ├── BUILD │ │ ├── ConversionUtils.scala │ │ ├── CountMetric.scala │ │ ├── EasyMetric.scala │ │ ├── FeatureCache.scala │ │ ├── LastResetMetric.scala │ │ ├── LatestMetric.scala │ │ ├── MaxMetric.scala │ │ ├── SumLikeMetric.scala │ │ ├── SumMetric.scala │ │ ├── SumSqMetric.scala │ │ ├── TimedValue.scala │ │ └── TimedValueAggregationMetric.scala │ ├── package.scala │ ├── query/ │ │ ├── BUILD │ │ └── ScopedAggregateBuilder.scala │ └── scalding/ │ ├── AggregateFeaturesMerger.scala │ ├── AggregatesStoreComparisonJob.scala │ ├── AggregatesV2ScaldingJob.scala │ ├── AggregationKeyOrdering.scala │ ├── BUILD │ ├── DeletedUserPruner.scala │ ├── MostRecentVersionedStore.scala │ └── sources/ │ ├── BUILD │ └── ScaldingAggregateSource.scala ├── topic-social-proof/ │ ├── README.md │ └── server/ │ ├── BUILD │ └── src/ │ └── main/ │ ├── resources/ │ │ ├── BUILD │ │ ├── config/ │ │ │ └── decider.yml │ │ └── logback.xml │ ├── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── tsp/ │ │ ├── BUILD │ │ ├── TopicSocialProofStratoFedServer.scala │ │ ├── columns/ │ │ │ ├── BUILD │ │ │ ├── TopicSocialProofBatchColumn.scala │ │ │ └── TopicSocialProofColumn.scala │ │ ├── common/ │ │ │ ├── BUILD │ │ │ ├── DeciderConstants.scala │ │ │ ├── FeatureSwitchesBuilder.scala │ │ │ ├── LoadShedder.scala │ │ │ ├── ParamsBuilder.scala │ │ │ ├── RecTargetFactory.scala │ │ │ ├── TopicSocialProofDecider.scala │ │ │ └── TopicSocialProofParams.scala │ │ ├── handlers/ │ │ │ ├── BUILD │ │ │ ├── TopicSocialProofHandler.scala │ │ │ └── UttChildrenWarmupHandler.scala │ │ ├── modules/ │ │ │ ├── BUILD │ │ │ ├── GizmoduckUserModule.scala │ │ │ ├── RepresentationScorerStoreModule.scala │ │ │ ├── TSPClientIdModule.scala │ │ │ ├── TopicListingModule.scala │ │ │ ├── TopicSocialProofStoreModule.scala │ │ │ ├── TopicTweetCosineSimilarityAggregateStoreModule.scala │ │ │ ├── TweetInfoStoreModule.scala │ │ │ ├── TweetyPieClientModule.scala │ │ │ ├── UnifiedCacheClient.scala │ │ │ ├── UttClientModule.scala │ │ │ └── UttLocalizationModule.scala │ │ ├── service/ │ │ │ ├── BUILD │ │ │ └── TopicSocialProofService.scala │ │ ├── stores/ │ │ │ ├── BUILD │ │ │ ├── LocalizedUttRecommendableTopicsStore.scala │ │ │ ├── RepresentationScorerStore.scala │ │ │ ├── SemanticCoreAnnotationStore.scala │ │ │ ├── TopicSocialProofStore.scala │ │ │ ├── TopicStore.scala │ │ │ ├── TopicTweetsCosineSimilarityAggregateStore.scala │ │ │ ├── TweetInfoStore.scala │ │ │ └── UttTopicFilterStore.scala │ │ └── utils/ │ │ ├── BUILD │ │ ├── LZ4Injection.scala │ │ ├── ReadableStoreWithMapOptionValues.scala │ │ └── SeqObjectInjection.scala │ └── thrift/ │ ├── BUILD │ ├── service.thrift │ └── tweet_info.thrift ├── trust_and_safety_models/ │ ├── README.md │ ├── abusive/ │ │ └── abusive_model.py │ ├── nsfw/ │ │ ├── nsfw_media.py │ │ └── nsfw_text.py │ └── toxicity/ │ ├── __init__.py │ ├── data/ │ │ ├── __init__.py │ │ ├── data_preprocessing.py │ │ ├── dataframe_loader.py │ │ └── mb_generator.py │ ├── load_model.py │ ├── optim/ │ │ ├── __init__.py │ │ ├── callbacks.py │ │ ├── losses.py │ │ └── schedulers.py │ ├── rescoring.py │ ├── settings/ │ │ ├── __init__.py │ │ └── default_settings_tox.py │ ├── train.py │ └── utils/ │ ├── __init__.py │ └── helpers.py ├── tweet-mixer/ │ └── server/ │ └── src/ │ └── main/ │ └── scala/ │ └── com/ │ └── twitter/ │ └── tweet_mixer/ │ ├── BUILD.bazel │ ├── TweetMixerHttpServerWarmupHandler.scala │ ├── TweetMixerServer.scala │ ├── TweetMixerThriftServerWarmupHandler.scala │ ├── candidate_pipeline/ │ │ ├── BUILD.bazel │ │ ├── CertoTopicTweetsCandidatePipelineConfigFactory.scala │ │ ├── ContentAnnTweetBasedCandidatePipelineConfigFactory.scala │ │ ├── ContentExplorationDRTweetTweetCandidatePipelineConfigFactory.scala │ │ ├── ContentExplorationDRTweetTweetTierTwoCandidatePipelineConfigFactory.scala │ │ ├── ContentExplorationDRUserTweetCandidatePipelineConfig.scala │ │ ├── ContentExplorationDRUserTweetTierTwoCandidatePipelineConfig.scala │ │ ├── ContentExplorationEmbeddingSimilarityCandidatePipelineConfigFactory.scala │ │ ├── ContentExplorationEmbeddingSimilarityTierTwoCandidatePipelineConfigFactory.scala │ │ ├── ContentExplorationEvergreenDRTweetTweetCandidatePipelineConfigFactory.scala │ │ ├── ContentExplorationSimclusterColdCandidatePipelineConfig.scala │ │ ├── ControlAiTopicCandidatePipelineConfig.scala │ │ ├── CuratedUserTlsPerLangaugeCandidatePipelineConfigFactory.scala │ │ ├── DeepRetrievalTweetTweetEmbeddingSimilarityCandidatePipelineConfigFactory.scala │ │ ├── DeepRetrievalTweetTweetSimilarityCandidatePipelineConfigFactory.scala │ │ ├── DeepRetrievalUserTweetSimilarityCandidatePipelineConfig.scala │ │ ├── EarlybirdInNetworkCandidatePipelineConfigFactory.scala │ │ ├── EventsCandidatePipelineConfig.scala │ │ ├── EvergreenDRCrossBorderUserTweetCandidatePipelineConfig.scala │ │ ├── EvergreenDRUserTweetCandidatePipelineConfig.scala │ │ ├── EvergreenVideosCandidatePipelineConfigFactory.scala │ │ ├── HaploliteCandidatePipelineConfigFactory.scala │ │ ├── MediaDeepRetrievalTweetTweetSimilarityCandidatePipelineConfigFactory.scala │ │ ├── MediaDeepRetrievalUserTweetSimilarityCandidatePipelineConfigFactory.scala │ │ ├── PinnedTweetRelatedCreatorPipelineConfigFactory.scala │ │ ├── PopGrokTopicTweetsCandidatePipelineConfigFactory.scala │ │ ├── PopularGeoTweetsCandidatePipelineConfigFactory.scala │ │ ├── PopularTopicTweetsCandidatePipelineConfigFactory.scala │ │ ├── QigSearchHistoryTweetsCandidatePipelineConfigFactory.scala │ │ ├── SemanticVideoCandidatePipelineConfigFactory.scala │ │ ├── SimclustersInterestedInCandidatePipelineConfigFactory.scala │ │ ├── SimclustersProducerBasedCandidatePipelineConfigFactory.scala │ │ ├── SimclustersPromotedCreatorCandidatePipelineConfigFactory.scala │ │ ├── SimclustersTweetBasedCandidatePipelineConfigFactory.scala │ │ ├── SkitTopicTweetsCandidatePipelineConfigFactory.scala │ │ ├── TrendsVideoCandidatePipelineConfig.scala │ │ ├── TwHINRebuildTweetSimilarityCandidatePipelineConfigFactory.scala │ │ ├── TwHINTweetSimilarityCandidatePipelineConfigFactory.scala │ │ ├── TwhinConsumerBasedCandidatePipelineConfigFactory.scala │ │ ├── TwhinUserTweetSimilarityCandidatePipelineConfig.scala │ │ ├── TwitterClipV0LongVideoCandidatePipelineConfigFactory.scala │ │ ├── TwitterClipV0ShortVideoCandidatePipelineConfigFactory.scala │ │ ├── UTEGCandidatePipelineConfigFactory.scala │ │ ├── UTGExpansionTweetBasedCandidatePipelineConfigFactory.scala │ │ ├── UTGProducerBasedCandidatePipelineConfigFactory.scala │ │ ├── UTGTweetBasedCandidatePipelineConfigFactory.scala │ │ ├── UVGExpansionTweetBasedCandidatePipelineConfigFactory.scala │ │ ├── UVGTweetBasedCandidatePipelineConfigFactory.scala │ │ ├── UserInterestsSummaryCandidatePipelineConfigFactory.scala │ │ └── UserLocationCandidatePipelineConfig.scala │ ├── candidate_source/ │ │ ├── BUILD.bazel │ │ ├── UTG/ │ │ │ ├── BUILD.bazel │ │ │ ├── UTGProducerBasedRequest.scala │ │ │ ├── UTGTweetBasedRequest.scala │ │ │ ├── UserTweetGraphConsumerBasedCandidateSource.scala │ │ │ ├── UserTweetGraphProducerBasedCandidateSource.scala │ │ │ └── UserTweetGraphTweetBasedCandidateSource.scala │ │ ├── UVG/ │ │ │ ├── BUILD.bazel │ │ │ ├── UVGTweetBasedRequest.scala │ │ │ ├── UserVideoGraphConsumerBasedCandidateSource.scala │ │ │ └── UserVideoGraphTweetBasedCandidateSource.scala │ │ ├── cached_candidate_source/ │ │ │ ├── BUILD.bazel │ │ │ └── MemcachedCandidateSource.scala │ │ ├── content_embedding_ann/ │ │ │ ├── BUILD.bazel │ │ │ └── ContentEmbeddingAnnCandidateSource.scala │ │ ├── curated_user_tls_per_language/ │ │ │ ├── BUILD.bazel │ │ │ └── CuratedUserTlsPerLanguageCandidateSource.scala │ │ ├── earlybird_realtime_cg/ │ │ │ ├── BUILD.bazel │ │ │ ├── EarlybirdRealtimeCGTweetCandidateSource.scala │ │ │ └── InNetworkRequest.scala │ │ ├── engaged_users/ │ │ │ ├── BUILD.bazel │ │ │ └── RecentEngagedUsersCandidateSource.scala │ │ ├── events/ │ │ │ ├── BUILD.bazel │ │ │ └── EventsCandidateSource.scala │ │ ├── evergreen_videos/ │ │ │ ├── BUILD.bazel │ │ │ ├── EvergreenVideosSearchByTweetQuery.scala │ │ │ ├── EvergreenVideosSearchByUserIdsQuery.scala │ │ │ ├── HistoricalEvergreenVideosCandidateSource.scala │ │ │ ├── MemeVideoCandidateSource.scala │ │ │ ├── SemanticVideoCandidateSource.scala │ │ │ ├── TwitterClipV0LongVideoCandidateSource.scala │ │ │ └── TwitterClipV0ShortVideoCandidateSource.scala │ │ ├── ndr_ann/ │ │ │ ├── BUILD.bazel │ │ │ ├── DRANNKey.scala │ │ │ ├── DRMultipleANNQuery.scala │ │ │ ├── DeepRetrievalTweetTweetANNCandidateSource.scala │ │ │ ├── DeepRetrievalTweetTweetEmbeddingANNCandidateSource.scala │ │ │ ├── DeepRetrievalUserTweetANNCandidateSource.scala │ │ │ ├── EmbeddingANNCandidateSource.scala │ │ │ ├── MediaDeepRetrievalUserTweetANNCandidateSource.scala │ │ │ └── UserInterestANNCandidateSource.scala │ │ ├── popular_geo_tweets/ │ │ │ ├── BUILD.bazel │ │ │ ├── PopularGeoTweetsCandidateSource.scala │ │ │ └── TripStratoGeoQuery.scala │ │ ├── popular_grok_topic_tweets/ │ │ │ ├── BUILD.bazel │ │ │ ├── GrokTopicTweetsQuery.scala │ │ │ └── PopGrokTopicTweetsCandidateSource.scala │ │ ├── popular_topic_tweets/ │ │ │ ├── BUILD.bazel │ │ │ ├── PopularTopicTweetsCandidateSource.scala │ │ │ └── TripStratoTopicQuery.scala │ │ ├── qig_service/ │ │ │ ├── BUILD.bazel │ │ │ ├── QigServiceBatchTweetCandidateSource.scala │ │ │ └── QigTweetCandidate.scala │ │ ├── simclusters_ann/ │ │ │ ├── BUILD.bazel │ │ │ ├── SANNQuery.scala │ │ │ ├── SimClustersAnnCandidateSource.scala │ │ │ ├── SimclusterColdPostsCandidateSource.scala │ │ │ ├── SimclusterColdPostsQuery.scala │ │ │ └── SimclusterPromotedCreatorAnnCandidateSource.scala │ │ ├── text_embedding_ann/ │ │ │ ├── BUILD.bazel │ │ │ ├── TextEmbeddingCandidateSource.scala │ │ │ └── TextEmbeddingQuery.scala │ │ ├── topic_tweets/ │ │ │ ├── BUILD.bazel │ │ │ ├── CertoTopicTweetsCandidateSource.scala │ │ │ └── SkitTopicTweetsCandidateSource.scala │ │ ├── trends/ │ │ │ ├── BUILD.bazel │ │ │ ├── TrendsCandidateSource.scala │ │ │ └── TrendsVideoCandidateSource.scala │ │ ├── twhin_ann/ │ │ │ ├── BUILD.bazel │ │ │ ├── TwHINANNCandidateSource.scala │ │ │ ├── TwHINRebuildANNCandidateSource.scala │ │ │ └── TwHINRebuildANNKey.scala │ │ ├── user_location/ │ │ │ ├── BUILD.bazel │ │ │ └── UserLocationCandidateSource.scala │ │ └── uss_service/ │ │ ├── BUILD.bazel │ │ └── USSSignalCandidateSource.scala │ ├── config/ │ │ ├── BUILD.bazel │ │ ├── SimClustersANNConfig.scala │ │ └── TimeoutConfig.scala │ ├── controller/ │ │ ├── BUILD.bazel │ │ └── TweetMixerThriftController.scala │ ├── feature/ │ │ ├── BUILD.bazel │ │ ├── EntityTypes.scala │ │ ├── FromInNetworkSourceFeature.scala │ │ ├── HydraScoreFeature.scala │ │ ├── InReplyToTweetIdFeature.scala │ │ ├── LanguageCodeFeature.scala │ │ ├── LowSignalUserFeature.scala │ │ ├── MediaMetadataFeatures.scala │ │ ├── PredictionRequestIdFeature.scala │ │ ├── RealGraphInNetworkScoresFeature.scala │ │ ├── RequestCountryPlaceIdFeature.scala │ │ ├── ScoreFeature.scala │ │ ├── SignalInfo.scala │ │ ├── SourceSignalFeature.scala │ │ ├── SourceTweetIdFeature.scala │ │ ├── TopicTweetScore.scala │ │ ├── TripTweetScore.scala │ │ ├── TweetInfoFeatures.scala │ │ ├── TweetTopicIdFeature.scala │ │ ├── USSFeatures.scala │ │ └── UserTopicIdsFeature.scala │ ├── functional_component/ │ │ ├── BUILD.bazel │ │ ├── TweetMixerFunctionalComponents.scala │ │ ├── filter/ │ │ │ ├── BUILD.bazel │ │ │ ├── GrokFilter.scala │ │ │ ├── ImpressedTweetsBloomFilter.scala │ │ │ ├── ImpressedTweetsFilter.scala │ │ │ ├── IsLongFormVideoFilter.scala │ │ │ ├── IsPortraitVideoFilter.scala │ │ │ ├── IsShortFormVideoFilter.scala │ │ │ ├── IsVideoTweetFilter.scala │ │ │ ├── MaxViewCountFilter.scala │ │ │ ├── MediaClusterIdDedupFilter.scala │ │ │ ├── MediaIdDedupFilter.scala │ │ │ ├── MediaWatchHistoryFilter.scala │ │ │ ├── MinScoreFilter.scala │ │ │ ├── ShouldIgnoreCandidatePipelinesFilter.scala │ │ │ └── TweetVisibilityAndReplyFilter.scala │ │ ├── gate/ │ │ │ ├── AllowLowSignalUserGate.scala │ │ │ ├── AllowNonEmptySearchHistoryUserGate.scala │ │ │ ├── BUILD.bazel │ │ │ ├── DenyLowSignalUserGate.scala │ │ │ ├── MaxFollowersGate.scala │ │ │ ├── MinTimeSinceLastRequestGate.scala │ │ │ └── ProbablisticPassGate.scala │ │ ├── hydrator/ │ │ │ ├── BUILD.bazel │ │ │ ├── ContentEmbeddingQueryFeatureHydratorFactory.scala │ │ │ ├── ContentExplorationDRUserEmbeddingQueryFeatureHydrator.scala │ │ │ ├── ContentMediaEmbeddingQueryFeatureHydratorFactory.scala │ │ │ ├── ControlAiTopicEmbeddingQueryFeatureHydrator.scala │ │ │ ├── DeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory.scala │ │ │ ├── DeepRetrievalUserEmbeddingQueryFeatureHydrator.scala │ │ │ ├── EvergreenDRUserEmbeddingQueryFeatureHydrator.scala │ │ │ ├── FeedbackHistoryQueryFeatureHydrator.scala │ │ │ ├── GizmoduckQueryFeatureHydrator.scala │ │ │ ├── GrokBooleanFeatureHydrator.scala │ │ │ ├── GrokCategoriesFeatureHydrator.scala │ │ │ ├── GrokFilterFeatureHydrator.scala │ │ │ ├── HaploliteQueryFeatureHydrator.scala │ │ │ ├── HighQualitySourceSignalQueryFeatureHydrator.scala │ │ │ ├── HydraRankingPreparationQueryFeatureHydrator.scala │ │ │ ├── ImpressionBloomFilterQueryFeatureHydrator.scala │ │ │ ├── ImpressionVideoBloomFilterVideoFeatureHydrator.scala │ │ │ ├── LastNonPollingTimeQueryFeatureHydrator.scala │ │ │ ├── MediaDeepRetrievalTweetEmbeddingQueryFeatureHydratorFactory.scala │ │ │ ├── MediaDeepRetrievalUserEmbeddingQueryFeatureHydrator.scala │ │ │ ├── MediaMetadataCandidateFeatureHydrator.scala │ │ │ ├── MultimodalEmbeddingQueryFeatureHydratorFactory.scala │ │ │ ├── OutlierDeepRetrievalEmbeddingQueryFeatureHydratorFactory.scala │ │ │ ├── RealGraphInNetworkScoresQueryFeatureHydrator.scala │ │ │ ├── RequestCountryPlaceIdFeatureHydrator.scala │ │ │ ├── SGSFollowedUsersQueryFeatureHydrator.scala │ │ │ ├── SignalInfoCandidateFeatureHydrator.scala │ │ │ ├── TweetypieCandidateFeatureHydrator.scala │ │ │ ├── TweetypieSeedTweetsQueryFeatureHydratorFactory.scala │ │ │ ├── TwhinUserPositiveEmbeddingQueryFeatureHydrator.scala │ │ │ ├── USSDeepRetrievalTweetEmbeddingFeatureHydrator.scala │ │ │ ├── USSGrokCategoryFeatureHydrator.scala │ │ │ ├── USSQueryFeatureHydrator.scala │ │ │ ├── UTGOutlierSignalsQueryFeatureHydrator.scala │ │ │ ├── UVGOutlierSignalsQueryFeatureHydrator.scala │ │ │ ├── UecAggTweetTotalFeatureHydrator.scala │ │ │ ├── UserInterestSummaryQueryFeatureHydrator.scala │ │ │ ├── UserSignalQueryFeatureHydrator.scala │ │ │ └── UserTopicIdsFeatureHydrator.scala │ │ ├── selector/ │ │ │ ├── BUILD.bazel │ │ │ ├── FavoriteSelector.scala │ │ │ ├── FeedbackRelevantSelector.scala │ │ │ ├── HydraBasedSorterProvider.scala │ │ │ ├── HydraBasedTransformedSorterProvider.scala │ │ │ ├── InsertAppendWeightedSignalPriorityWeaveResults.scala │ │ │ ├── ReserveVideoSelector.scala │ │ │ └── UprankVideoSorterProvider.scala │ │ ├── side_effect/ │ │ │ ├── BUILD.bazel │ │ │ ├── DeepRetrievalAdHocSideEffect.scala │ │ │ ├── EvergreenVideosSideEffect.scala │ │ │ ├── HydraScoringSideEffect.scala │ │ │ ├── PublishGroxUserInterestsSideEffect.scala │ │ │ ├── RequestMultimodalEmbeddingSideEffect.scala │ │ │ ├── ScribeServedCandidatesSideEffect.scala │ │ │ └── SelectedStatsSideEffect.scala │ │ └── transformer/ │ │ ├── AnnCandidateFeatureTransformer.scala │ │ ├── BUILD.bazel │ │ ├── CertoTopicTweetsQueryTransformer.scala │ │ ├── EarlybirdInNetworkQueryTransformer.scala │ │ ├── EarlybirdInNetworkResponseFeatureTransformer.scala │ │ ├── EvergreenVideosQueryTransformer.scala │ │ ├── EvergreenVideosResponseFeatureTransformer.scala │ │ ├── GrokTopicTweetsQueryTransformer.scala │ │ ├── HaploliteResponseFeatureTransformer.scala │ │ ├── QigBatchQueryTransformer.scala │ │ ├── QigTweetCandidateFeatureTransformer.scala │ │ ├── SANNQueryTransformer.scala │ │ ├── SkitTopicTweetsQueryTransformer.scala │ │ ├── TimelineQueryTransformer.scala │ │ ├── TopicTweetFeatureTransformer.scala │ │ ├── TripStratoGeoQueryTransformer.scala │ │ ├── TripStratoTopicQueryTransformer.scala │ │ ├── TripTweetFeatureTransformer.scala │ │ ├── TweetFeatureTimelineServiceTransformer.scala │ │ ├── TweetMixerCandidateFeatureTransformer.scala │ │ ├── TwitterClipV0LongVideoQueryTransformer.scala │ │ ├── TwitterClipV0ShortVideoQueryTransformer.scala │ │ ├── UTGProducerBasedQueryTransformer.scala │ │ ├── UTGTweetBasedQueryTransformer.scala │ │ ├── UVGTweetBasedQueryTransformer.scala │ │ ├── UtegQueryTransformer.scala │ │ └── UtegResponseFeatureTransformer.scala │ ├── marshaller/ │ │ ├── request/ │ │ │ ├── BUILD.bazel │ │ │ ├── TweetMixerDebugParamsUnmarshaller.scala │ │ │ ├── TweetMixerProductContextUnmarshaller.scala │ │ │ ├── TweetMixerProductUnmarshaller.scala │ │ │ └── TweetMixerRequestUnmarshaller.scala │ │ └── response/ │ │ ├── BUILD.bazel │ │ ├── TweetMixerProductResponseMarshaller.scala │ │ ├── TweetMixerResponseTransportMarshaller.scala │ │ └── common/ │ │ ├── BUILD.bazel │ │ └── TweetResultMarshaller.scala │ ├── model/ │ │ ├── BUILD.bazel │ │ ├── ModuleNames.scala │ │ ├── request/ │ │ │ ├── BUILD.bazel │ │ │ ├── HasContentCategory.scala │ │ │ ├── HasTopicIds.scala │ │ │ ├── HasVideoType.scala │ │ │ ├── TweetMixerDebugOptions.scala │ │ │ ├── TweetMixerProduct.scala │ │ │ └── TweetMixerRequest.scala │ │ └── response/ │ │ ├── BUILD.bazel │ │ ├── RecommendationResult.scala │ │ ├── TweetMixerCandidate.scala │ │ ├── TweetMixerProductResponse.scala │ │ └── TweetMixerResponse.scala │ ├── module/ │ │ ├── BUILD.bazel │ │ ├── CertoStratoTopicTweetsStoreModule.scala │ │ ├── ExtendedStratoClientModule.scala │ │ ├── GPURetrievalHttpClientModule.scala │ │ ├── HaploliteClientModule.scala │ │ ├── HydraEmbeddingGenerationServiceClientModule.scala │ │ ├── HydraRootClientModule.scala │ │ ├── InMemoryCacheModule.scala │ │ ├── MHMtlsParamsModule.scala │ │ ├── ManhattanFeatureRepositoryModule.scala │ │ ├── MemCacheClientModule.scala │ │ ├── PipelineFailureExceptionMapper.scala │ │ ├── SampleFeatureStoreV1DynamicClientBuilderModule.scala │ │ ├── SimClustersANNServiceNameToClientMapper.scala │ │ ├── SkitStratoTopicTweetsStoreModule.scala │ │ ├── StitchMemcacheClientModule.scala │ │ ├── TimeoutConfigModule.scala │ │ ├── TwHINANNServiceModule.scala │ │ ├── TwHINEmbeddingStoreModule.scala │ │ ├── TweetMixerFlagModule.scala │ │ ├── UserStateStoreModule.scala │ │ └── thrift_client/ │ │ ├── AnnEmbeddingProducerModule.scala │ │ ├── AnnQueryServiceClientModule.scala │ │ ├── AnnQueryableByIdModule.scala │ │ ├── BUILD.bazel │ │ ├── EarlybirdRealtimeCGModule.scala │ │ ├── GeoduckHydrationClientModule.scala │ │ ├── GeoduckLocationServiceClientModule.scala │ │ ├── QigServiceClientModule.scala │ │ ├── SimClustersAnnServiceClientModule.scala │ │ ├── TweetyPieClientModule.scala │ │ ├── UserTweetGraphClientModule.scala │ │ ├── UserVideoGraphClientModule.scala │ │ └── VecDBAnnServiceClientModule.scala │ ├── param/ │ │ ├── BUILD.bazel │ │ ├── CandidateSourceParams.scala │ │ ├── CertoTopicTweetsParams.scala │ │ ├── ContentEmbeddingAnnParams.scala │ │ ├── CuratedUserTlsPerLanguageParams.scala │ │ ├── EarlybirdInNetworkTweetsParams.scala │ │ ├── EvergreenParams.scala │ │ ├── GlobalParamConfigModule.scala │ │ ├── HighQualitySourceSignalParams.scala │ │ ├── PopGrokTopicTweetsParams.scala │ │ ├── PopularGeoTweetsParams.scala │ │ ├── PopularTopicTweetsParams.scala │ │ ├── SimClustersAnnParams.scala │ │ ├── SkitTopicTweetsParams.scala │ │ ├── TweetMixerGlobalParamConfig.scala │ │ ├── TweetMixerGlobalParams.scala │ │ ├── USSParams.scala │ │ ├── UTGParams.scala │ │ ├── UVGParams.scala │ │ ├── UserLocationParams.scala │ │ └── decider/ │ │ ├── BUILD.bazel │ │ └── DeciderKey.scala │ ├── product/ │ │ ├── BUILD.bazel │ │ ├── TweetMixerProductModule.scala │ │ ├── TweetMixerProductPipelineRegistryConfig.scala │ │ └── home_recommended_tweets/ │ │ ├── BUILD.bazel │ │ ├── HomeRecommendedTweetsProductPipelineConfig.scala │ │ ├── HomeRecommendedTweetsRecommendationPipelineConfig.scala │ │ ├── marshaller/ │ │ │ ├── request/ │ │ │ │ ├── BUILD.bazel │ │ │ │ └── HomeRecommendedTweetsProductContextUnmarshaller.scala │ │ │ └── response/ │ │ │ ├── BUILD.bazel │ │ │ ├── HomeRecommendedTweetsDomainResponseMarshaller.scala │ │ │ └── HomeRecommendedTweetsProductResponseMarshaller.scala │ │ ├── model/ │ │ │ ├── request/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── HomeRecommendedTweetsProductContext.scala │ │ │ │ └── HomeRecommendedTweetsQuery.scala │ │ │ └── response/ │ │ │ ├── BUILD.bazel │ │ │ ├── HomeRecommendedTweetsProductResponse.scala │ │ │ └── HomeRecommendedTweetsResult.scala │ │ └── param/ │ │ ├── BUILD.bazel │ │ ├── HomeRecommendedTweetsParam.scala │ │ └── HomeRecommendedTweetsParamConfig.scala │ ├── scorer/ │ │ ├── BUILD.bazel │ │ └── HydraScorer.scala │ ├── scoring_pipeline/ │ │ ├── BUILD.bazel │ │ └── HydraScoringPipelineConfig.scala │ ├── service/ │ │ ├── BUILD.bazel │ │ ├── TweetMixerAccessPolicy.scala │ │ ├── TweetMixerNotificationConfig.scala │ │ └── TweetMixerService.scala │ ├── store/ │ │ ├── BUILD.bazel │ │ └── TwhinEmbeddingsStore.scala │ └── utils/ │ ├── BUILD.bazel │ ├── BucketSnowflakeIdAgeStats.scala │ ├── CandidatePipelineConstants.scala │ ├── CandidateSourceUtil.scala │ ├── ConcurrentMapCache.scala │ ├── InjectionTransformer.scala │ ├── MemCacheStitchClient.scala │ ├── PipelineFailureCategories.scala │ ├── SignalUtils.scala │ ├── Transformers.scala │ └── Utils.scala ├── tweetypie/ │ ├── common/ │ │ └── src/ │ │ ├── scala/ │ │ │ └── com/ │ │ │ └── twitter/ │ │ │ └── tweetypie/ │ │ │ ├── additionalfields/ │ │ │ │ ├── AdditionalFields.scala │ │ │ │ └── BUILD │ │ │ ├── caching/ │ │ │ │ ├── BUILD │ │ │ │ ├── CacheOperations.scala │ │ │ │ ├── CacheResult.scala │ │ │ │ ├── Expiry.scala │ │ │ │ ├── ServoCachedValueSerializer.scala │ │ │ │ ├── SoftTtl.scala │ │ │ │ ├── StitchAsync.scala │ │ │ │ ├── StitchCacheOperations.scala │ │ │ │ ├── StitchCaching.scala │ │ │ │ └── ValueSerializer.scala │ │ │ ├── client_id/ │ │ │ │ ├── BUILD │ │ │ │ └── ClientIdHelper.scala │ │ │ ├── context/ │ │ │ │ ├── BUILD │ │ │ │ └── TweetypieContext.scala │ │ │ ├── decider/ │ │ │ │ ├── BUILD │ │ │ │ ├── DeciderGates.scala │ │ │ │ └── overrides/ │ │ │ │ ├── BUILD │ │ │ │ └── TweetyPieDeciderOverrides.scala │ │ │ ├── jiminy/ │ │ │ │ └── tweetypie/ │ │ │ │ ├── BUILD │ │ │ │ └── NudgeBuilder.scala │ │ │ ├── matching/ │ │ │ │ ├── BUILD │ │ │ │ ├── TokenSequence.scala │ │ │ │ ├── Tokenizer.scala │ │ │ │ ├── TweetTokenizer.scala │ │ │ │ └── UserMutes.scala │ │ │ ├── media/ │ │ │ │ ├── BUILD │ │ │ │ ├── Media.scala │ │ │ │ ├── MediaUrl.scala │ │ │ │ └── package.scala │ │ │ ├── storage/ │ │ │ │ ├── AddTweetHandler.scala │ │ │ │ ├── BUILD │ │ │ │ ├── BounceDeleteHandler.scala │ │ │ │ ├── Codecs.scala │ │ │ │ ├── DeleteAdditionalFieldsHandler.scala │ │ │ │ ├── Field.scala │ │ │ │ ├── GetDeletedTweetsHandler.scala │ │ │ │ ├── GetStoredTweetHandler.scala │ │ │ │ ├── GetTweetHandler.scala │ │ │ │ ├── HardDeleteTweetHandler.scala │ │ │ │ ├── InspectFields.scala │ │ │ │ ├── Json.scala │ │ │ │ ├── ManhattanOperations.scala │ │ │ │ ├── ManhattanTweetStorageClient.scala │ │ │ │ ├── Response.scala │ │ │ │ ├── Scribe.scala │ │ │ │ ├── ScrubHandler.scala │ │ │ │ ├── SoftDeleteHandler.scala │ │ │ │ ├── Stats.scala │ │ │ │ ├── StatusConversions.scala │ │ │ │ ├── StorageConversions.scala │ │ │ │ ├── TimestampDecoder.scala │ │ │ │ ├── TweetKey.scala │ │ │ │ ├── TweetStateRecord.scala │ │ │ │ ├── TweetStorageClient.scala │ │ │ │ ├── TweetStorageException.scala │ │ │ │ ├── TweetUtils.scala │ │ │ │ ├── UndeleteHandler.scala │ │ │ │ ├── UpdateTweetHandler.scala │ │ │ │ └── package.scala │ │ │ ├── tflock/ │ │ │ │ ├── BUILD │ │ │ │ ├── TFlockIndexer.scala │ │ │ │ └── TweetIndexer.scala │ │ │ ├── thriftscala/ │ │ │ │ ├── BUILD │ │ │ │ ├── NotImplementedTweetService.scala │ │ │ │ ├── TweetServiceProxy.scala │ │ │ │ └── entities/ │ │ │ │ ├── BUILD │ │ │ │ ├── CashtagTextEntity.scala │ │ │ │ ├── EntityExtractor.scala │ │ │ │ ├── HashtagTextEntity.scala │ │ │ │ ├── Implicits.scala │ │ │ │ ├── MediaTextEntity.scala │ │ │ │ ├── MentionTextEntity.scala │ │ │ │ ├── TextRangeEntityAdapter.scala │ │ │ │ └── UrlTextEntity.scala │ │ │ ├── tweettext/ │ │ │ │ ├── BUILD │ │ │ │ ├── GraphemeIndexIterator.scala │ │ │ │ ├── IndexConverter.scala │ │ │ │ ├── Offset.scala │ │ │ │ ├── PartialHtmlEncoding.scala │ │ │ │ ├── Preprocessor.scala │ │ │ │ ├── TextEntity.scala │ │ │ │ ├── TextModification.scala │ │ │ │ ├── Truncator.scala │ │ │ │ └── TweetText.scala │ │ │ └── util/ │ │ │ ├── BUILD │ │ │ ├── CommunityAnnotation.scala │ │ │ ├── CommunityUtil.scala │ │ │ ├── ConversationControls.scala │ │ │ ├── EditControlUtil.scala │ │ │ ├── RetryPolicyBuilder.scala │ │ │ ├── StitchUtils.scala │ │ │ ├── StringLiteral.scala │ │ │ ├── Takedowns.scala │ │ │ ├── TransientContextUtil.scala │ │ │ ├── TweetCreationLock.scala │ │ │ ├── TweetLenses.scala │ │ │ ├── TweetPermalinkUtil.scala │ │ │ ├── TweetTransformer.scala │ │ │ ├── logging/ │ │ │ │ ├── AlertableExceptionLoggingFilter.scala │ │ │ │ ├── BUILD │ │ │ │ └── OnlyImportantLogsLoggingFilter.scala │ │ │ └── package.scala │ │ └── thrift/ │ │ └── com/ │ │ └── twitter/ │ │ └── tweetypie/ │ │ ├── BUILD │ │ ├── api_fields.thrift │ │ ├── creative-entity-enrichments/ │ │ │ └── creative_entity_enrichments.thrift │ │ ├── delete_location_data.thrift │ │ ├── deleted_tweet.thrift │ │ ├── deprecated.thrift │ │ ├── edit_control.thrift │ │ ├── geo/ │ │ │ └── tweet_location_info.thrift │ │ ├── media/ │ │ │ └── media_ref.thrift │ │ ├── media_entity.thrift │ │ ├── note_tweet.thrift │ │ ├── retweet_archival_event.thrift │ │ ├── storage_internal/ │ │ │ ├── BUILD │ │ │ └── storage_internal.thrift │ │ ├── stored_tweet_info.thrift │ │ ├── transient_context.thrift │ │ ├── tweet.thrift │ │ ├── tweet_audit.thrift │ │ ├── tweet_comparison_service.thrift │ │ ├── tweet_events.thrift │ │ ├── tweet_service.thrift │ │ ├── tweet_service_federated.thrift │ │ ├── tweet_service_graphql.thrift │ │ └── unmentions/ │ │ └── unmentions.thrift │ ├── server/ │ │ ├── BUILD │ │ ├── README.md │ │ ├── config/ │ │ │ ├── BUILD │ │ │ ├── decider.yml │ │ │ ├── decider_staging.yml │ │ │ ├── logging/ │ │ │ │ ├── logback-all-include.xml │ │ │ │ ├── logback-without-loglens.xml │ │ │ │ └── logback.xml │ │ │ └── partner_media.yml │ │ └── src/ │ │ └── main/ │ │ ├── scala/ │ │ │ └── com/ │ │ │ └── twitter/ │ │ │ └── tweetypie/ │ │ │ ├── BUILD │ │ │ ├── backends/ │ │ │ │ ├── BUILD │ │ │ │ ├── Backend.scala │ │ │ │ ├── ConfigBus.scala │ │ │ │ ├── CreativesContainerService.scala │ │ │ │ ├── Escherbird.scala │ │ │ │ ├── Expandodo.scala │ │ │ │ ├── GeoScrubEventStore.scala │ │ │ │ ├── Gizmoduck.scala │ │ │ │ ├── GnipEnricherator.scala │ │ │ │ ├── LimiterBackend.scala │ │ │ │ ├── LimiterService.scala │ │ │ │ ├── Manhattan.scala │ │ │ │ ├── MediaInfoService.scala │ │ │ │ ├── Scarecrow.scala │ │ │ │ ├── SocialGraphService.scala │ │ │ │ ├── TFlock.scala │ │ │ │ ├── Talon.scala │ │ │ │ ├── TimelineService.scala │ │ │ │ ├── UserImageService.scala │ │ │ │ └── Warmup.scala │ │ │ ├── config/ │ │ │ │ ├── BUILD │ │ │ │ ├── BackendClients.scala │ │ │ │ ├── Caches.scala │ │ │ │ ├── ClientsParser.scala │ │ │ │ ├── DynamicConfig.scala │ │ │ │ ├── DynamicConfigLoader.scala │ │ │ │ ├── ExternalRepositories.scala │ │ │ │ ├── LogicalRepositories.scala │ │ │ │ ├── Main.scala │ │ │ │ ├── MemcacheExceptionLoggingFilter.scala │ │ │ │ ├── Resources.scala │ │ │ │ ├── ScribeTweetCacheWrites.scala │ │ │ │ ├── TweetBuilders.scala │ │ │ │ ├── TweetHydrators.scala │ │ │ │ ├── TweetServerBuilder.scala │ │ │ │ ├── TweetServiceAuthorizers.scala │ │ │ │ ├── TweetServiceBuilder.scala │ │ │ │ ├── TweetServiceInvocationBuilder.scala │ │ │ │ ├── TweetServiceSettings.scala │ │ │ │ ├── TweetStores.scala │ │ │ │ ├── TweetypieDeciderGates.scala │ │ │ │ ├── WritePathHydration.scala │ │ │ │ └── package.scala │ │ │ ├── core/ │ │ │ │ ├── BUILD │ │ │ │ ├── CardReferenceUriExtractor.scala │ │ │ │ ├── EditState.scala │ │ │ │ ├── Exceptions.scala │ │ │ │ ├── FilteredState.scala │ │ │ │ ├── GeoSearchRequestId.scala │ │ │ │ ├── HydrationState.scala │ │ │ │ ├── QuotedTweetResult.scala │ │ │ │ ├── Serializer.scala │ │ │ │ ├── StoredTweetResult.scala │ │ │ │ ├── TweetCreateFailure.scala │ │ │ │ ├── TweetData.scala │ │ │ │ ├── TweetResult.scala │ │ │ │ ├── UpstreamFailure.scala │ │ │ │ ├── ValueState.scala │ │ │ │ └── package.scala │ │ │ ├── federated/ │ │ │ │ ├── BUILD │ │ │ │ ├── StratoCatalogBuilder.scala │ │ │ │ ├── columns/ │ │ │ │ │ ├── AccessPolicy.scala │ │ │ │ │ ├── ApiErrors.scala │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── CreateRetweetColumn.scala │ │ │ │ │ ├── CreateTweetColumn.scala │ │ │ │ │ ├── DeleteTweetColumn.scala │ │ │ │ │ ├── FederatedFieldColumn.scala │ │ │ │ │ ├── FederatedFieldGroup.scala │ │ │ │ │ ├── FederatedFieldReq.scala │ │ │ │ │ ├── GetStoredTweetsByUserColumn.scala │ │ │ │ │ ├── GetStoredTweetsColumn.scala │ │ │ │ │ ├── GetTweetFieldsColumn.scala │ │ │ │ │ ├── HydrationOptions.scala │ │ │ │ │ ├── TrackingId.scala │ │ │ │ │ ├── TweetypieContactInfo.scala │ │ │ │ │ └── UnretweetColumn.scala │ │ │ │ ├── context/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── RequestContext.scala │ │ │ │ ├── prefetcheddata/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── PrefetchedDataRepository.scala │ │ │ │ ├── promotedcontent/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── TweetPromotedContentLogger.scala │ │ │ │ └── warmups/ │ │ │ │ ├── BUILD │ │ │ │ └── StratoCatalogWarmups.scala │ │ │ ├── handler/ │ │ │ │ ├── AttachmentBuilder.scala │ │ │ │ ├── BUILD │ │ │ │ ├── CardReferenceValidationHandler.scala │ │ │ │ ├── CardUsersFinder.scala │ │ │ │ ├── CollabControlBuilder.scala │ │ │ │ ├── CommunitiesValidator.scala │ │ │ │ ├── ConversationControlBuilder.scala │ │ │ │ ├── DeleteAdditionalFieldsBuilder.scala │ │ │ │ ├── DeleteLocationDataHandler.scala │ │ │ │ ├── DuplicateTweetFinder.scala │ │ │ │ ├── EditControlBuilder.scala │ │ │ │ ├── EditValidator.scala │ │ │ │ ├── EraseUserTweetsHandler.scala │ │ │ │ ├── GeoBuilder.scala │ │ │ │ ├── GetDeletedTweetsHandler.scala │ │ │ │ ├── GetStoredTweetsByUserHandler.scala │ │ │ │ ├── GetStoredTweetsHandler.scala │ │ │ │ ├── GetTweetCountsHandler.scala │ │ │ │ ├── GetTweetFieldsHandler.scala │ │ │ │ ├── GetTweetsHandler.scala │ │ │ │ ├── HandlerError.scala │ │ │ │ ├── MediaBuilder.scala │ │ │ │ ├── PostTweet.scala │ │ │ │ ├── QuotedTweetDeleteEventBuilder.scala │ │ │ │ ├── QuotedTweetTakedownEventBuilder.scala │ │ │ │ ├── RateLimitChecker.scala │ │ │ │ ├── ReplyBuilder.scala │ │ │ │ ├── RetweetBuilder.scala │ │ │ │ ├── ReverseGeocoder.scala │ │ │ │ ├── ScarecrowRetweetSpamChecker.scala │ │ │ │ ├── ScarecrowTweetSpamChecker.scala │ │ │ │ ├── ScrubGeoEventBuilder.scala │ │ │ │ ├── SelfThreadBuilder.scala │ │ │ │ ├── SetAdditionalFieldsBuilder.scala │ │ │ │ ├── SetRetweetVisibilityHandler.scala │ │ │ │ ├── Spam.scala │ │ │ │ ├── TakedownHandler.scala │ │ │ │ ├── TweetBuilder.scala │ │ │ │ ├── TweetCreationLock.scala │ │ │ │ ├── TweetDeletePathHandler.scala │ │ │ │ ├── TweetWriteValidator.scala │ │ │ │ ├── U13ValidationUtil.scala │ │ │ │ ├── UndeleteTweetHandler.scala │ │ │ │ ├── UnretweetHandler.scala │ │ │ │ ├── UpdatePossiblySensitiveTweetHandler.scala │ │ │ │ ├── UrlEntityBuilder.scala │ │ │ │ ├── UrlShortener.scala │ │ │ │ ├── UserTakedownHandler.scala │ │ │ │ ├── WritePathQueryOptions.scala │ │ │ │ └── package.scala │ │ │ ├── hydrator/ │ │ │ │ ├── BUILD │ │ │ │ ├── Card2Hydrator.scala │ │ │ │ ├── CardHydrator.scala │ │ │ │ ├── ContributorHydrator.scala │ │ │ │ ├── ContributorVisibilityFilter.scala │ │ │ │ ├── ConversationControlHydrator.scala │ │ │ │ ├── ConversationIdHydrator.scala │ │ │ │ ├── ConversationMutedHydrator.scala │ │ │ │ ├── CopyFromSourceTweet.scala │ │ │ │ ├── CreatedAtRepairer.scala │ │ │ │ ├── DeviceSourceHydrator.scala │ │ │ │ ├── DirectedAtHydrator.scala │ │ │ │ ├── EditControlHydrator.scala │ │ │ │ ├── EditHydrator.scala │ │ │ │ ├── EditPerspectiveHydrator.scala │ │ │ │ ├── EscherbirdAnnotationHydrator.scala │ │ │ │ ├── FeatureSwitchResultsHydrator.scala │ │ │ │ ├── GeoScrubHydrator.scala │ │ │ │ ├── HasMediaHydrator.scala │ │ │ │ ├── IM1837FilterHydrator.scala │ │ │ │ ├── IM2884FilterHydrator.scala │ │ │ │ ├── IM3433FilterHydrator.scala │ │ │ │ ├── LanguageHydrator.scala │ │ │ │ ├── MediaEntityHydrator.scala │ │ │ │ ├── MediaInfoHydrator.scala │ │ │ │ ├── MediaIsProtectedHydrator.scala │ │ │ │ ├── MediaKeyHydrator.scala │ │ │ │ ├── MediaRefsHydrator.scala │ │ │ │ ├── MediaTagsHydrator.scala │ │ │ │ ├── MediaUrlFieldsHydrator.scala │ │ │ │ ├── MentionEntityHydrator.scala │ │ │ │ ├── NegativeVisibleTextRangeRepairer.scala │ │ │ │ ├── NoteTweetSuffixHydrator.scala │ │ │ │ ├── PartialEntityCleaner.scala │ │ │ │ ├── PastedMediaHydrator.scala │ │ │ │ ├── PerspectiveHydrator.scala │ │ │ │ ├── PlaceHydrator.scala │ │ │ │ ├── PreviousTweetCountsHydrator.scala │ │ │ │ ├── ProfileGeoHydrator.scala │ │ │ │ ├── QuoteTweetVisibilityHydrator.scala │ │ │ │ ├── QuotedTweetHydrator.scala │ │ │ │ ├── QuotedTweetRefHydrator.scala │ │ │ │ ├── QuotedTweetRefUrlsHydrator.scala │ │ │ │ ├── RepairMutation.scala │ │ │ │ ├── ReplyScreenNameHydrator.scala │ │ │ │ ├── ReportedTweetFilter.scala │ │ │ │ ├── RetweetMediaRepairer.scala │ │ │ │ ├── RetweetParentStatusIdRepairer.scala │ │ │ │ ├── ScrubEngagementHydrator.scala │ │ │ │ ├── ScrubUncacheableTweetRepairer.scala │ │ │ │ ├── SourceTweetHydrator.scala │ │ │ │ ├── StripHiddenGeoCoordinates.scala │ │ │ │ ├── SuperfluousUrlEntityScrubber.scala │ │ │ │ ├── TakedownHydrator.scala │ │ │ │ ├── TextRepairer.scala │ │ │ │ ├── TweetAuthorVisibilityHydrator.scala │ │ │ │ ├── TweetCountsHydrator.scala │ │ │ │ ├── TweetCtx.scala │ │ │ │ ├── TweetHydration.scala │ │ │ │ ├── TweetLegacyFormatter.scala │ │ │ │ ├── TweetQueryOptionsExpander.scala │ │ │ │ ├── TweetVisibilityHydrator.scala │ │ │ │ ├── UnmentionDataHydrator.scala │ │ │ │ ├── UnrequestedFieldScrubber.scala │ │ │ │ ├── UrlEntityHydrator.scala │ │ │ │ ├── ValueHydrator.scala │ │ │ │ └── package.scala │ │ │ ├── media/ │ │ │ │ ├── BUILD │ │ │ │ ├── MediaClient.scala │ │ │ │ ├── MediaKeyClassifier.scala │ │ │ │ ├── MediaKeyUtil.scala │ │ │ │ └── MediaMetadata.scala │ │ │ ├── package.scala │ │ │ ├── repository/ │ │ │ │ ├── BUILD │ │ │ │ ├── CacheStitch.scala │ │ │ │ ├── CachingTweetRepository.scala │ │ │ │ ├── Card2Repository.scala │ │ │ │ ├── CardRepository.scala │ │ │ │ ├── CardUsersRepository.scala │ │ │ │ ├── ConversationControlRepository.scala │ │ │ │ ├── ConversationIdRepository.scala │ │ │ │ ├── ConversationMutedRepository.scala │ │ │ │ ├── CreativesContainerMaterializationRepository.scala │ │ │ │ ├── DeletedTweetVisibilityRepository.scala │ │ │ │ ├── DeviceSourceRepository.scala │ │ │ │ ├── EscherbirdAnnotationRepository.scala │ │ │ │ ├── GeoScrubTimestampRepository.scala │ │ │ │ ├── GeoduckPlaceRepository.scala │ │ │ │ ├── LastQuoteOfQuoterRepository.scala │ │ │ │ ├── ManhattanTweetRepository.scala │ │ │ │ ├── MediaMetadataRepository.scala │ │ │ │ ├── ParentUserIdRepository.scala │ │ │ │ ├── PastedMediaRepository.scala │ │ │ │ ├── PenguinLanguageRepository.scala │ │ │ │ ├── PerspectiveRepository.scala │ │ │ │ ├── PlaceRepository.scala │ │ │ │ ├── ProfileGeoRepository.scala │ │ │ │ ├── QuotedTweetVisibilityRepository.scala │ │ │ │ ├── QuoterHasAlreadyQuotedRepository.scala │ │ │ │ ├── RelationshipRepository.scala │ │ │ │ ├── RetweetSpamCheckRepository.scala │ │ │ │ ├── StitchLockingCache.scala │ │ │ │ ├── StratoCommunityAccessRepository.scala │ │ │ │ ├── StratoCommunityMembershipRepository.scala │ │ │ │ ├── StratoPromotedTweetRepository.scala │ │ │ │ ├── StratoSafetyLabelsRepository.scala │ │ │ │ ├── StratoSubscriptionVerificationRepository.scala │ │ │ │ ├── StratoSuperFollowEligibleRepository.scala │ │ │ │ ├── StratoSuperFollowRelationsRepository.scala │ │ │ │ ├── TweetCountsRepository.scala │ │ │ │ ├── TweetQuery.scala │ │ │ │ ├── TweetRepository.scala │ │ │ │ ├── TweetResultRepository.scala │ │ │ │ ├── TweetSpamCheckRepository.scala │ │ │ │ ├── TweetVisibilityRepository.scala │ │ │ │ ├── UnmentionInfoRepository.scala │ │ │ │ ├── UnmentionedEntitiesRepository.scala │ │ │ │ ├── UrlRepository.scala │ │ │ │ ├── UserInfoRepository.scala │ │ │ │ ├── UserRepository.scala │ │ │ │ ├── UserTakedownRepository.scala │ │ │ │ ├── UserViewerRecipient.scala │ │ │ │ ├── VibeRepository.scala │ │ │ │ ├── VisibilityResultToFilteredState.scala │ │ │ │ └── package.scala │ │ │ ├── serverutil/ │ │ │ │ ├── ActivityService.scala │ │ │ │ ├── ActivityUtil.scala │ │ │ │ ├── BUILD │ │ │ │ ├── BoringStackTrace.scala │ │ │ │ ├── CaffeineMemcacheClient.scala │ │ │ │ ├── DeviceSourceParser.scala │ │ │ │ ├── ExceptionCounter.scala │ │ │ │ ├── ExtendedTweetMetadataBuilder.scala │ │ │ │ ├── NullMemcacheClient.scala │ │ │ │ ├── PartnerMedia.scala │ │ │ │ ├── StoredCard.scala │ │ │ │ └── logcachewrites/ │ │ │ │ ├── BUILD │ │ │ │ ├── TweetCacheWrite.scala │ │ │ │ └── WriteLoggingCache.scala │ │ │ ├── service/ │ │ │ │ ├── BUILD │ │ │ │ ├── ClientHandlingTweetService.scala │ │ │ │ ├── DispatchingTweetService.scala │ │ │ │ ├── FailureLoggingTweetService.scala │ │ │ │ ├── MethodAuthorizer.scala │ │ │ │ ├── ObservedTweetService.scala │ │ │ │ ├── QuillTweetService.scala │ │ │ │ ├── ReplicatingTweetService.scala │ │ │ │ ├── RescueExceptions.scala │ │ │ │ ├── TweetServiceProxy.scala │ │ │ │ ├── TweetServiceWarmer.scala │ │ │ │ ├── observer/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── GetDeletedTweetsObserver.scala │ │ │ │ │ ├── GetStoredTweetsByUserObserver.scala │ │ │ │ │ ├── GetStoredTweetsObserver.scala │ │ │ │ │ ├── GetTweetCountsObserver.scala │ │ │ │ │ ├── GetTweetFieldsObserver.scala │ │ │ │ │ ├── GetTweetsObserver.scala │ │ │ │ │ ├── Observer.scala │ │ │ │ │ ├── PostTweetObserver.scala │ │ │ │ │ ├── ResultStateStats.scala │ │ │ │ │ ├── StoredTweetsObserver.scala │ │ │ │ │ └── package.scala │ │ │ │ └── package.scala │ │ │ └── store/ │ │ │ ├── AsyncEnqueueStore.scala │ │ │ ├── BUILD │ │ │ ├── CachingTweetStore.scala │ │ │ ├── DeleteAdditionalFields.scala │ │ │ ├── DeleteTweet.scala │ │ │ ├── FanoutServiceStore.scala │ │ │ ├── Flush.scala │ │ │ ├── GeoSearchRequestIDStore.scala │ │ │ ├── GizmoduckUserCountsUpdatingStore.scala │ │ │ ├── GizmoduckUserGeotagUpdateStore.scala │ │ │ ├── Guano.scala │ │ │ ├── GuanoServiceStore.scala │ │ │ ├── IncrBookmarkCount.scala │ │ │ ├── IncrFavCount.scala │ │ │ ├── InitialTweetUpdate.scala │ │ │ ├── InsertTweet.scala │ │ │ ├── LimiterStore.scala │ │ │ ├── LogLensStore.scala │ │ │ ├── ManhattanTweetStore.scala │ │ │ ├── MediaIndexHelper.scala │ │ │ ├── MediaServiceStore.scala │ │ │ ├── QuotedTweetDelete.scala │ │ │ ├── QuotedTweetOps.scala │ │ │ ├── QuotedTweetTakedown.scala │ │ │ ├── ReplicatingTweetStore.scala │ │ │ ├── RetweetArchivalEnqueueStore.scala │ │ │ ├── ScribeMediaTagStore.scala │ │ │ ├── ScrubGeo.scala │ │ │ ├── SetAdditionalFields.scala │ │ │ ├── SetRetweetVisibility.scala │ │ │ ├── Takedown.scala │ │ │ ├── TlsTimelineUpdatingStore.scala │ │ │ ├── TweetCountsCacheUpdatingStore.scala │ │ │ ├── TweetEventBusStore.scala │ │ │ ├── TweetIndexingStore.scala │ │ │ ├── TweetStatsStore.scala │ │ │ ├── TweetStore.scala │ │ │ ├── TweetStoreEvent.scala │ │ │ ├── TweetUpdate.scala │ │ │ ├── UndeleteTweet.scala │ │ │ ├── UpdatePossiblySensitiveTweet.scala │ │ │ └── package.scala │ │ └── thrift/ │ │ ├── BUILD │ │ └── tweetypie_internal.thrift │ └── servo/ │ ├── README.md │ ├── decider/ │ │ ├── BUILD │ │ └── src/ │ │ └── main/ │ │ └── scala/ │ │ ├── BUILD │ │ └── com/ │ │ └── twitter/ │ │ └── servo/ │ │ ├── decider/ │ │ │ ├── DeciderGateBuilder.scala │ │ │ ├── DeciderKeyEnum.scala │ │ │ └── package.scala │ │ └── gate/ │ │ └── DeciderGate.scala │ ├── json/ │ │ ├── BUILD │ │ └── src/ │ │ └── main/ │ │ └── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── servo/ │ │ └── json/ │ │ ├── BUILD │ │ └── ThriftJsonInspector.scala │ ├── repo/ │ │ ├── BUILD │ │ └── src/ │ │ └── main/ │ │ ├── scala/ │ │ │ ├── BUILD │ │ │ └── com/ │ │ │ └── twitter/ │ │ │ └── servo/ │ │ │ ├── cache/ │ │ │ │ ├── ByteCountingMemcache.scala │ │ │ │ ├── Cache.scala │ │ │ │ ├── CacheFactory.scala │ │ │ │ ├── Cached.scala │ │ │ │ ├── CounterCache.scala │ │ │ │ ├── CounterSerializer.scala │ │ │ │ ├── FinagleMemcache.scala │ │ │ │ ├── ForwardingCache.scala │ │ │ │ ├── HotKeyMemcacheClient.scala │ │ │ │ ├── InProcessCache.scala │ │ │ │ ├── IterableSerializer.scala │ │ │ │ ├── KeyFilteringCache.scala │ │ │ │ ├── KeyTransformer.scala │ │ │ │ ├── LockingCache.scala │ │ │ │ ├── Memcache.scala │ │ │ │ ├── MigratingCache.scala │ │ │ │ ├── MissingCache.scala │ │ │ │ ├── ObservableCache.scala │ │ │ │ ├── SecondaryIndexingCache.scala │ │ │ │ ├── SelectedCache.scala │ │ │ │ ├── SeqSerializer.scala │ │ │ │ ├── Serializer.scala │ │ │ │ ├── SetSerializer.scala │ │ │ │ ├── SimpleReplicatingCache.scala │ │ │ │ ├── TransformingCache.scala │ │ │ │ ├── TtlCache.scala │ │ │ │ └── package.scala │ │ │ ├── database/ │ │ │ │ ├── Accessors.scala │ │ │ │ ├── Bitfield.scala │ │ │ │ ├── Credentials.scala │ │ │ │ ├── Database.scala │ │ │ │ └── package.scala │ │ │ ├── hydrator/ │ │ │ │ └── KeyValueHydrator.scala │ │ │ ├── keyvalue/ │ │ │ │ └── KeyValueResult.scala │ │ │ ├── repository/ │ │ │ │ ├── CachingCounterKeyValueRepository.scala │ │ │ │ ├── CachingKeyValueRepository.scala │ │ │ │ ├── ChunkingStrategy.scala │ │ │ │ ├── DarkmodingKeyValueRepositoryFactory.scala │ │ │ │ ├── HotKeyCachingKeyValueRepository.scala │ │ │ │ ├── ImmutableKeyValueRepository.scala │ │ │ │ ├── KeyValueRepository.scala │ │ │ │ ├── ObservableKeyValueRepository.scala │ │ │ │ ├── Repository.scala │ │ │ │ ├── ResponseCachingKeyValueRepository.scala │ │ │ │ ├── RichQuery.scala │ │ │ │ ├── SuccessRateTrackingRepository.scala │ │ │ │ └── package.scala │ │ │ └── store/ │ │ │ ├── CachingStore.scala │ │ │ ├── KeyValueStore.scala │ │ │ ├── ObservableStore.scala │ │ │ └── Store.scala │ │ └── thrift/ │ │ ├── BUILD │ │ └── com/ │ │ └── twitter/ │ │ └── servo/ │ │ └── cache/ │ │ └── servo_repo.thrift │ ├── request/ │ │ ├── BUILD │ │ └── src/ │ │ └── main/ │ │ └── scala/ │ │ ├── BUILD │ │ └── com/ │ │ └── twitter/ │ │ └── servo/ │ │ └── request/ │ │ ├── ClientRequestAuthorizer.scala │ │ ├── ClientRequestObserver.scala │ │ ├── PermissionModule.scala │ │ ├── RequestFilter.scala │ │ ├── RequestHandler.scala │ │ └── package.scala │ └── util/ │ ├── BUILD │ └── src/ │ └── main/ │ └── scala/ │ ├── BUILD │ └── com/ │ └── twitter/ │ └── servo/ │ ├── data/ │ │ ├── Lens.scala │ │ └── Mutation.scala │ ├── forked/ │ │ ├── Forked.scala │ │ └── QueueExecutor.scala │ ├── gate/ │ │ └── RateLimitingGate.scala │ └── util/ │ ├── Availability.scala │ ├── Average.scala │ ├── BatchExecutor.scala │ ├── CancelledExceptionExtractor.scala │ ├── CounterInitializingStatsReceiver.scala │ ├── Effect.scala │ ├── ExceptionCounter.scala │ ├── FrequencyCounter.scala │ ├── FunctionArrow.scala │ ├── FutureArrow.scala │ ├── FutureEffect.scala │ ├── Gate.scala │ ├── LogarithmicallyBucketedTimer.scala │ ├── MemoizingStatsReceiver.scala │ ├── Observable.scala │ ├── OptionOrdering.scala │ ├── RandomPerturber.scala │ ├── RateLimitingLogger.scala │ ├── Retry.scala │ ├── RetryHandler.scala │ ├── RpcRetry.scala │ ├── Scribe.scala │ ├── SuccessRateTracker.scala │ ├── SynchronizedHashMap.scala │ ├── ThreadLocalStringBuilder.scala │ ├── ThrowableHelper.scala │ ├── Transformer.scala │ ├── TryOrdering.scala │ ├── WaitForServerSets.scala │ └── package.scala ├── twml/ │ ├── BUILD │ ├── README.md │ ├── libtwml/ │ │ ├── BUILD │ │ ├── include/ │ │ │ ├── twml/ │ │ │ │ ├── BatchPredictionRequest.h │ │ │ │ ├── BatchPredictionResponse.h │ │ │ │ ├── BlockFormatReader.h │ │ │ │ ├── BlockFormatWriter.h │ │ │ │ ├── DataRecord.h │ │ │ │ ├── DataRecordReader.h │ │ │ │ ├── DataRecordWriter.h │ │ │ │ ├── Error.h │ │ │ │ ├── HashedDataRecord.h │ │ │ │ ├── HashedDataRecordReader.h │ │ │ │ ├── Hashmap.h │ │ │ │ ├── RawTensor.h │ │ │ │ ├── Tensor.h │ │ │ │ ├── TensorRecord.h │ │ │ │ ├── TensorRecordReader.h │ │ │ │ ├── TensorRecordWriter.h │ │ │ │ ├── ThriftReader.h │ │ │ │ ├── ThriftWriter.h │ │ │ │ ├── Type.h │ │ │ │ ├── common.h │ │ │ │ ├── defines.h │ │ │ │ ├── discretizer_impl.h │ │ │ │ ├── functions.h │ │ │ │ ├── hashing_discretizer_impl.h │ │ │ │ ├── io/ │ │ │ │ │ └── IOError.h │ │ │ │ ├── optim.h │ │ │ │ └── utilities.h │ │ │ └── twml.h │ │ ├── setup.cfg │ │ ├── setup.py │ │ └── src/ │ │ ├── lib/ │ │ │ ├── BatchPredictionRequest.cpp │ │ │ ├── BatchPredictionResponse.cpp │ │ │ ├── BlockFormatReader.cpp │ │ │ ├── BlockFormatWriter.cpp │ │ │ ├── CMakeLists.txt │ │ │ ├── CPPLINT.cfg │ │ │ ├── DataRecord.cpp │ │ │ ├── DataRecordReader.cpp │ │ │ ├── DataRecordWriter.cpp │ │ │ ├── HashedDataRecord.cpp │ │ │ ├── HashedDataRecordReader.cpp │ │ │ ├── Hashmap.cpp │ │ │ ├── Tensor.cpp │ │ │ ├── TensorRecordReader.cpp │ │ │ ├── TensorRecordWriter.cpp │ │ │ ├── ThriftReader.cpp │ │ │ ├── ThriftWriter.cpp │ │ │ ├── discretizer_impl.cpp │ │ │ ├── functions.cpp │ │ │ ├── hashing_discretizer_impl.cpp │ │ │ ├── internal/ │ │ │ │ ├── endianutils.h │ │ │ │ ├── error.h │ │ │ │ ├── interpolate.h │ │ │ │ ├── khash.h │ │ │ │ ├── linear_search.h │ │ │ │ ├── murmur_hash3.h │ │ │ │ ├── thrift.h │ │ │ │ └── utf_converter.h │ │ │ ├── io/ │ │ │ │ └── IOError.cpp │ │ │ ├── murmur_hash3.cpp │ │ │ ├── optim.cpp │ │ │ └── utf_converter.cpp │ │ └── ops/ │ │ ├── CMakeLists.txt │ │ ├── add1.cpp │ │ ├── batch_prediction_request.cpp │ │ ├── batch_prediction_request_v2.cpp │ │ ├── batch_prediction_response_writer.cpp │ │ ├── batch_prediction_tensor_response_writer.cpp │ │ ├── binary_sparse_dense_matmul.cpp │ │ ├── binary_sparse_dense_matmul.h │ │ ├── binary_sparse_dense_matmul_impl.h │ │ ├── block_format_dataset.cpp │ │ ├── block_format_reader.h │ │ ├── compress_sample_ids.cpp │ │ ├── contrib/ │ │ │ └── get_substrings.cpp │ │ ├── data_record.cpp │ │ ├── data_record_tensor_writer.cpp │ │ ├── discretizer.cpp │ │ ├── feature_extractor.cpp │ │ ├── feature_id.cpp │ │ ├── feature_mask.cpp │ │ ├── fixed_length_tensor.cpp │ │ ├── hashed_data_record.cpp │ │ ├── hashing_discretizer.cpp │ │ ├── hashmap.cpp │ │ ├── isotonic_calibration.cpp │ │ ├── num_intra_op_threads.cpp │ │ ├── par_add.cpp │ │ ├── partition_sparse_tensor.cpp │ │ ├── percentile_discretizer_v2.cpp │ │ ├── resource_utils.h │ │ ├── scripts/ │ │ │ ├── get_inc.py │ │ │ ├── get_inc.sh │ │ │ ├── get_lib.py │ │ │ ├── get_lib.sh │ │ │ └── symlink.sh │ │ ├── sleep_op.cpp │ │ ├── sparse_normalization.cpp │ │ ├── tensor_record.cpp │ │ ├── tensorflow_utils.cpp │ │ ├── tensorflow_utils.h │ │ └── var_length_reader.cpp │ ├── setup.cfg │ ├── setup.py │ ├── twml/ │ │ ├── __init__.py │ │ ├── argument_parser.py │ │ ├── array.py │ │ ├── block_format_writer.py │ │ ├── constants.py │ │ ├── contrib/ │ │ │ ├── __init__.py │ │ │ ├── build_graphs_fns.py │ │ │ ├── calibrators/ │ │ │ │ ├── __init__.py │ │ │ │ ├── calibrator.py │ │ │ │ ├── common_calibrators.py │ │ │ │ ├── hashed_percentile_discretizer.py │ │ │ │ ├── hashing_discretizer.py │ │ │ │ ├── isotonic.py │ │ │ │ ├── mdl.py │ │ │ │ └── percentile_discretizer.py │ │ │ ├── eventbus/ │ │ │ │ ├── input_fn.py │ │ │ │ └── reader.py │ │ │ ├── export/ │ │ │ │ ├── __init__.py │ │ │ │ ├── export_fn.py │ │ │ │ └── exporters.py │ │ │ ├── feature_config.py │ │ │ ├── feature_config_parsers.py │ │ │ ├── feature_importances/ │ │ │ │ ├── __init__.py │ │ │ │ ├── feature_importances.py │ │ │ │ ├── feature_permutation.py │ │ │ │ └── helpers.py │ │ │ ├── hooks.py │ │ │ ├── initializers.py │ │ │ ├── layers/ │ │ │ │ ├── __init__.py │ │ │ │ ├── embedding_lookup.py │ │ │ │ ├── factorization_machine.py │ │ │ │ ├── full_dense.py │ │ │ │ ├── hashed_percentile_discretizer.py │ │ │ │ ├── hashing_discretizer.py │ │ │ │ ├── mask_layer.py │ │ │ │ ├── stacked_rnn.py │ │ │ │ └── zscore_normalization.py │ │ │ ├── metrics/ │ │ │ │ ├── __init__.py │ │ │ │ ├── metrics.py │ │ │ │ └── search_metrics.py │ │ │ ├── optimizers/ │ │ │ │ ├── __init__.py │ │ │ │ ├── deep_gradient_compression_optimizer.py │ │ │ │ └── pruning_optimizer.py │ │ │ ├── parsers.py │ │ │ ├── pruning.py │ │ │ ├── readers/ │ │ │ │ ├── __init__.py │ │ │ │ ├── batch_prediction_request.py │ │ │ │ ├── data_record.py │ │ │ │ └── hashed_batch_prediction_request.py │ │ │ ├── trainers/ │ │ │ │ ├── __init__.py │ │ │ │ ├── batch_prediction_request_trainer.py │ │ │ │ ├── pruning_data_record_trainer.py │ │ │ │ └── trainer_utils.py │ │ │ └── utils/ │ │ │ ├── __init__.py │ │ │ ├── datasets.py │ │ │ ├── device.py │ │ │ ├── interp.py │ │ │ ├── loss_fns.py │ │ │ ├── masks.py │ │ │ ├── math_fns.py │ │ │ ├── normalizer.py │ │ │ ├── scores.py │ │ │ └── similarities.py │ │ ├── dataset.py │ │ ├── errors.py │ │ ├── export_output_fns.py │ │ ├── feature_config.py │ │ ├── filters.py │ │ ├── hooks.py │ │ ├── input_fns.py │ │ ├── layers/ │ │ │ ├── __init__.py │ │ │ ├── batch_prediction_tensor_writer.py │ │ │ ├── batch_prediction_writer.py │ │ │ ├── data_record_tensor_writer.py │ │ │ ├── full_dense.py │ │ │ ├── full_sparse.py │ │ │ ├── isotonic.py │ │ │ ├── layer.py │ │ │ ├── mdl.py │ │ │ ├── partition.py │ │ │ ├── percentile_discretizer.py │ │ │ ├── sequential.py │ │ │ ├── sparse_max_norm.py │ │ │ └── stitch.py │ │ ├── learning_rate_decay.py │ │ ├── lookup/ │ │ │ └── __init__.py │ │ ├── metrics.py │ │ ├── optimizers/ │ │ │ └── __init__.py │ │ ├── parsers.py │ │ ├── readers/ │ │ │ ├── __init__.py │ │ │ ├── batch_prediction_request.py │ │ │ ├── data_record.py │ │ │ ├── hashed_batch_prediction_request.py │ │ │ └── hashed_data_record.py │ │ ├── saved_model_cli/ │ │ │ ├── __init__.py │ │ │ └── __main__.py │ │ ├── summary/ │ │ │ └── __init__.py │ │ ├── tensorboard/ │ │ │ ├── __init__.py │ │ │ └── __main__.py │ │ ├── tensorio.py │ │ ├── tracking/ │ │ │ ├── __init__.py │ │ │ └── experiment_tracker.py │ │ ├── trainers/ │ │ │ ├── __init__.py │ │ │ ├── data_record_trainer.py │ │ │ └── trainer.py │ │ └── util.py │ └── twml_common/ │ ├── __init__.py │ ├── initializer.py │ ├── serialize.py │ └── sparse_inputs.py ├── unified_user_actions/ │ ├── .gitignore │ ├── BUILD.bazel │ ├── README.md │ ├── adapter/ │ │ └── src/ │ │ ├── main/ │ │ │ └── scala/ │ │ │ └── com/ │ │ │ └── twitter/ │ │ │ └── unified_user_actions/ │ │ │ └── adapter/ │ │ │ ├── AbstractAdapter.scala │ │ │ ├── BUILD │ │ │ ├── ads_callback_engagements/ │ │ │ │ ├── AdsCallbackEngagement.scala │ │ │ │ ├── AdsCallbackEngagementsAdapter.scala │ │ │ │ ├── BUILD │ │ │ │ ├── BaseAdsCallbackEngagement.scala │ │ │ │ ├── BaseTrendAdsCallbackEngagement.scala │ │ │ │ ├── BaseVideoAdsCallbackEngagement.scala │ │ │ │ ├── EngagementTypeMappings.scala │ │ │ │ └── ProfileAdsCallbackEngagement.scala │ │ │ ├── client_event/ │ │ │ │ ├── BUILD │ │ │ │ ├── BaseCTAClientEvent.scala │ │ │ │ ├── BaseCardClientEvent.scala │ │ │ │ ├── BaseClientEvent.scala │ │ │ │ ├── BaseFeedbackSubmitClientEvent.scala │ │ │ │ ├── BaseNotificationTabClientEvent.scala │ │ │ │ ├── BaseProfileClientEvent.scala │ │ │ │ ├── BasePushNotificationClientEvent.scala │ │ │ │ ├── BaseSearchTypeaheadEvent.scala │ │ │ │ ├── BaseTopicClientEvent.scala │ │ │ │ ├── BaseUASClientEvent.scala │ │ │ │ ├── BaseVideoClientEvent.scala │ │ │ │ ├── ClientEventAdapter.scala │ │ │ │ ├── ClientEventCommonUtils.scala │ │ │ │ ├── ClientEventEngagement.scala │ │ │ │ ├── ClientEventImpression.scala │ │ │ │ ├── HomeInfoUtils.scala │ │ │ │ ├── ItemTypeFilterPredicates.scala │ │ │ │ ├── NotificationClientEventUtils.scala │ │ │ │ ├── ProductSurfaceUtils.scala │ │ │ │ ├── SearchInfoUtils.scala │ │ │ │ ├── TopicIdUtils.scala │ │ │ │ └── VideoClientEventUtils.scala │ │ │ ├── common/ │ │ │ │ ├── AdapterUtils.scala │ │ │ │ └── BUILD │ │ │ ├── email_notification_event/ │ │ │ │ ├── BUILD │ │ │ │ ├── EmailNotificationEventAdapter.scala │ │ │ │ └── EmailNotificationEventUtils.scala │ │ │ ├── favorite_archival_events/ │ │ │ │ ├── BUILD │ │ │ │ └── FavoriteArchivalEventsAdapter.scala │ │ │ ├── retweet_archival_events/ │ │ │ │ ├── BUILD │ │ │ │ └── RetweetArchivalEventsAdapter.scala │ │ │ ├── social_graph_event/ │ │ │ │ ├── BUILD │ │ │ │ ├── BaseReportSocialGraphWriteEvent.scala │ │ │ │ ├── BaseSocialGraphWriteEvent.scala │ │ │ │ ├── SocialGraphAdapter.scala │ │ │ │ └── SocialGraphEngagement.scala │ │ │ ├── tls_favs_event/ │ │ │ │ ├── BUILD │ │ │ │ └── TlsFavsAdapter.scala │ │ │ ├── tweetypie_event/ │ │ │ │ ├── BUILD │ │ │ │ ├── BaseTweetypieTweetEvent.scala │ │ │ │ ├── BaseTweetypieTweetEventCreate.scala │ │ │ │ ├── BaseTweetypieTweetEventDelete.scala │ │ │ │ ├── TweetypieEventAdapter.scala │ │ │ │ └── TweetypieEventUtils.scala │ │ │ ├── user_modification_event/ │ │ │ │ ├── BUILD.bazel │ │ │ │ ├── UserModificationAdapter.scala │ │ │ │ └── UserModifications.scala │ │ │ └── uua_aggregates/ │ │ │ ├── BUILD │ │ │ ├── README │ │ │ ├── RekeyUuaAdapter.scala │ │ │ ├── RekeyUuaFromInteractionEventsAdapter.scala │ │ │ └── UuaActions.scala │ │ └── test/ │ │ └── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── unified_user_actions/ │ │ └── adapter/ │ │ ├── AdapterUtilsSpec.scala │ │ ├── AdsCallbackEngagementsAdapterSpec.scala │ │ ├── BUILD.bazel │ │ ├── ClientEventAdapterSpec.scala │ │ ├── EmailNotificationEventAdapterSpec.scala │ │ ├── EmailNotificationEventUtilsSpec.scala │ │ ├── FavoriteArchivalEventsAdapterSpec.scala │ │ ├── RekeyUuaFromInteractionEventsAdapterSpec.scala │ │ ├── RetweetArchivalEventsAdapterSpec.scala │ │ ├── SearchInfoUtilsSpec.scala │ │ ├── SocialGraphAdapterSpec.scala │ │ ├── TestFixtures.scala │ │ ├── TlsFavsAdapterSpec.scala │ │ ├── TopicsIdUtilsSpec.scala │ │ ├── TweetypieEventAdapterSpec.scala │ │ ├── UserModificationAdapterSpec.scala │ │ └── VideoClientEventUtilsSpec.scala │ ├── client/ │ │ └── src/ │ │ ├── main/ │ │ │ └── scala/ │ │ │ └── com/ │ │ │ └── twitter/ │ │ │ └── unified_user_actions/ │ │ │ └── client/ │ │ │ ├── config/ │ │ │ │ ├── BUILD │ │ │ │ ├── Clusters.scala │ │ │ │ ├── Constants.scala │ │ │ │ ├── Environments.scala │ │ │ │ └── KafkaConfigs.scala │ │ │ └── summingbird/ │ │ │ ├── BUILD │ │ │ └── UnifiedUserActionsSourceScrooge.scala │ │ └── test/ │ │ └── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── unified_user_actions/ │ │ └── client/ │ │ └── config/ │ │ ├── BUILD.bazel │ │ └── KafkaConfigsSpec.scala │ ├── enricher/ │ │ ├── BUILD.bazel │ │ ├── README.md │ │ └── src/ │ │ ├── main/ │ │ │ ├── scala/ │ │ │ │ └── com/ │ │ │ │ └── twitter/ │ │ │ │ └── unified_user_actions/ │ │ │ │ └── enricher/ │ │ │ │ ├── BUILD │ │ │ │ ├── Exceptions.scala │ │ │ │ ├── driver/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── EnrichmentDriver.scala │ │ │ │ │ └── EnrichmentPlanUtils.scala │ │ │ │ ├── graphql/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── GraphqlRspParser.scala │ │ │ │ ├── hcache/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── LocalCache.scala │ │ │ │ │ └── ObservedEvictingCache.scala │ │ │ │ ├── hydrator/ │ │ │ │ │ ├── AbstractHydrator.scala │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── DefaultHydrator.scala │ │ │ │ │ ├── Hydrator.scala │ │ │ │ │ └── NoopHydrator.scala │ │ │ │ └── partitioner/ │ │ │ │ ├── BUILD │ │ │ │ ├── DefaultPartitioner.scala │ │ │ │ └── Partitioner.scala │ │ │ └── thrift/ │ │ │ └── com/ │ │ │ └── twitter/ │ │ │ └── unified_user_actions/ │ │ │ └── enricher/ │ │ │ └── internal/ │ │ │ ├── BUILD │ │ │ ├── enrichment_envelop.thrift │ │ │ ├── enrichment_key.thrift │ │ │ └── enrichment_plan.thrift │ │ └── test/ │ │ ├── resources/ │ │ │ ├── BUILD.bazel │ │ │ └── logback.xml │ │ └── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── unified_user_actions/ │ │ └── enricher/ │ │ ├── BUILD.bazel │ │ ├── EnricherFixture.scala │ │ ├── driver/ │ │ │ ├── BUILD.bazel │ │ │ └── DriverTest.scala │ │ ├── graphql/ │ │ │ ├── BUILD.bazel │ │ │ └── GraphqlSpecs.scala │ │ ├── hcache/ │ │ │ ├── BUILD.bazel │ │ │ └── LocalCacheTest.scala │ │ ├── hydrator/ │ │ │ ├── BUILD.bazel │ │ │ ├── DefaultHydratorTest.scala │ │ │ └── NoopHydratorTest.scala │ │ └── partitioner/ │ │ ├── BUILD.bazel │ │ └── DefaultPartitionerTest.scala │ ├── graphql/ │ │ ├── README.md │ │ └── TweetHydration.graphql │ ├── kafka/ │ │ └── src/ │ │ ├── main/ │ │ │ └── scala/ │ │ │ └── com/ │ │ │ └── twitter/ │ │ │ └── unified_user_actions/ │ │ │ └── kafka/ │ │ │ ├── BUILD │ │ │ ├── ClientConfigs.scala │ │ │ ├── ClientProviders.scala │ │ │ ├── CompressionTypeFlag.scala │ │ │ └── serde/ │ │ │ ├── NullableScalaSerdes.scala │ │ │ └── internal/ │ │ │ └── thrift.scala │ │ └── test/ │ │ ├── resources/ │ │ │ ├── BUILD.bazel │ │ │ └── logback-test.xml │ │ └── scala/ │ │ ├── BUILD.bazel │ │ └── com/ │ │ └── twitter/ │ │ └── unified_user_actions/ │ │ └── kafka/ │ │ └── serde/ │ │ ├── NullableScalaSerdesSpec.scala │ │ └── TestLogAppender.scala │ ├── scripts/ │ │ └── kill_staging.sh │ ├── service/ │ │ ├── deploy/ │ │ │ ├── kill-staging-services.workflow │ │ │ ├── rekey-uua-iesource-prod.workflow │ │ │ ├── rekey-uua-iesource-staging.workflow │ │ │ ├── rekey-uua-iesource.aurora │ │ │ ├── rekey-uua-prod.workflow │ │ │ ├── rekey-uua-staging.workflow │ │ │ ├── rekey-uua.aurora │ │ │ ├── uua-ads-callback-engagements-prod.workflow │ │ │ ├── uua-ads-callback-engagements-staging.workflow │ │ │ ├── uua-ads-callback-engagements.aurora │ │ │ ├── uua-client-event-prod.workflow │ │ │ ├── uua-client-event-staging.workflow │ │ │ ├── uua-client-event.aurora │ │ │ ├── uua-email-notification-event-prod.workflow │ │ │ ├── uua-email-notification-event-staging.workflow │ │ │ ├── uua-email-notification-event.aurora │ │ │ ├── uua-enricher-staging.workflow │ │ │ ├── uua-enricher.aurora │ │ │ ├── uua-enrichment-planner-staging.workflow │ │ │ ├── uua-enrichment-planner.aurora │ │ │ ├── uua-favorite-archival-events-prod.workflow │ │ │ ├── uua-favorite-archival-events-staging.workflow │ │ │ ├── uua-favorite-archival-events.aurora │ │ │ ├── uua-retweet-archival-events-prod.workflow │ │ │ ├── uua-retweet-archival-events-staging.workflow │ │ │ ├── uua-retweet-archival-events.aurora │ │ │ ├── uua-social-graph-prod.workflow │ │ │ ├── uua-social-graph-staging.workflow │ │ │ ├── uua-social-graph.aurora │ │ │ ├── uua-tls-favs-prod.workflow │ │ │ ├── uua-tls-favs-staging.workflow │ │ │ ├── uua-tls-favs.aurora │ │ │ ├── uua-tweetypie-event-prod.workflow │ │ │ ├── uua-tweetypie-event-staging.workflow │ │ │ ├── uua-tweetypie-event.aurora │ │ │ ├── uua-user-modification-prod.workflow │ │ │ ├── uua-user-modification-staging.workflow │ │ │ └── uua-user-modification.aurora │ │ └── src/ │ │ ├── main/ │ │ │ ├── resources/ │ │ │ │ ├── BUILD │ │ │ │ ├── decider.yml │ │ │ │ └── logback.xml │ │ │ └── scala/ │ │ │ ├── BUILD │ │ │ └── com/ │ │ │ └── twitter/ │ │ │ └── unified_user_actions/ │ │ │ └── service/ │ │ │ ├── AdsCallbackEngagementsService.scala │ │ │ ├── BUILD │ │ │ ├── ClientEventService.scala │ │ │ ├── EmailNotificationEventService.scala │ │ │ ├── EnricherService.scala │ │ │ ├── EnrichmentPlannerService.scala │ │ │ ├── FavoriteArchivalEventsService.scala │ │ │ ├── RekeyUuaIesourceService.scala │ │ │ ├── RekeyUuaService.scala │ │ │ ├── RetweetArchivalEventsService.scala │ │ │ ├── SocialGraphService.scala │ │ │ ├── TlsFavsService.scala │ │ │ ├── TweetypieEventService.scala │ │ │ ├── UserModificationService.scala │ │ │ └── module/ │ │ │ ├── BUILD │ │ │ ├── CacheModule.scala │ │ │ ├── ClientIdModule.scala │ │ │ ├── DeciderUtils.scala │ │ │ ├── FlagsModule.scala │ │ │ ├── GraphqlClientProviderModule.scala │ │ │ ├── KafkaProcessorAdsCallbackEngagementsModule.scala │ │ │ ├── KafkaProcessorClientEventModule.scala │ │ │ ├── KafkaProcessorEmailNotificationEventModule.scala │ │ │ ├── KafkaProcessorFavoriteArchivalEventsModule.scala │ │ │ ├── KafkaProcessorProvider.scala │ │ │ ├── KafkaProcessorRekeyUuaIesourceModule.scala │ │ │ ├── KafkaProcessorRekeyUuaModule.scala │ │ │ ├── KafkaProcessorRetweetArchivalEventsModule.scala │ │ │ ├── KafkaProcessorSocialGraphModule.scala │ │ │ ├── KafkaProcessorTlsFavsModule.scala │ │ │ ├── KafkaProcessorTweetypieEventModule.scala │ │ │ ├── KafkaProcessorUserModificationModule.scala │ │ │ ├── TopicsMapping.scala │ │ │ └── ZoneFiltering.scala │ │ └── test/ │ │ ├── resources/ │ │ │ ├── BUILD.bazel │ │ │ ├── decider.yml │ │ │ └── logback.xml │ │ └── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── unified_user_actions/ │ │ └── service/ │ │ ├── BUILD.bazel │ │ ├── ClientEventServiceStartupTest.scala │ │ ├── DeciderUtilsTest.scala │ │ ├── EnrichmentPlannerServiceTest.scala │ │ ├── RekeyUuaIesourceServiceStartupTest.scala │ │ ├── TlsFavServiceStartupTest.scala │ │ └── ZoneFilteringTest.scala │ └── thrift/ │ └── src/ │ ├── main/ │ │ └── thrift/ │ │ └── com/ │ │ └── twitter/ │ │ └── unified_user_actions/ │ │ ├── BUILD │ │ ├── action_info.thrift │ │ ├── common.thrift │ │ ├── item.thrift │ │ ├── keyed_uua.thrift │ │ ├── metadata.thrift │ │ ├── product_surface_info.thrift │ │ └── unified_user_actions.thrift │ └── test/ │ └── thrift/ │ └── com/ │ └── twitter/ │ └── unified_user_actions/ │ ├── BUILD.bazel │ └── unified_user_actions.thrift ├── user-signal-service/ │ ├── README.md │ ├── server/ │ │ ├── BUILD │ │ └── src/ │ │ └── main/ │ │ ├── resources/ │ │ │ ├── BUILD │ │ │ ├── config/ │ │ │ │ └── decider.yml │ │ │ └── logback.xml │ │ └── scala/ │ │ └── com/ │ │ └── twitter/ │ │ └── usersignalservice/ │ │ ├── BUILD │ │ ├── UserSignalServiceStratoFedServerMain.scala │ │ ├── base/ │ │ │ ├── AggregatedSignalController.scala │ │ │ ├── BUILD │ │ │ ├── BaseSignalFetcher.scala │ │ │ ├── FilteredSignalFetcherController.scala │ │ │ ├── ManhattanSignalFetcher.scala │ │ │ ├── MemcachedSignalFetcherWrapper.scala │ │ │ └── StratoSignalFetcher.scala │ │ ├── columns/ │ │ │ ├── BUILD │ │ │ └── UserSignalServiceColumn.scala │ │ ├── config/ │ │ │ ├── BUILD │ │ │ └── SignalFetcherConfig.scala │ │ ├── handler/ │ │ │ ├── BUILD │ │ │ └── UserSignalHandler.scala │ │ ├── module/ │ │ │ ├── BUILD │ │ │ ├── CacheModule.scala │ │ │ ├── MHMtlsParamsModule.scala │ │ │ ├── SocialGraphServiceClientModule.scala │ │ │ └── TimerModule.scala │ │ ├── service/ │ │ │ ├── BUILD │ │ │ └── UserSignalService.scala │ │ └── signals/ │ │ ├── AccountBlocksFetcher.scala │ │ ├── AccountFollowsFetcher.scala │ │ ├── AccountMutesFetcher.scala │ │ ├── BUILD │ │ ├── NegativeEngagedTweetFetcher.scala │ │ ├── NegativeEngagedUserFetcher.scala │ │ ├── NotificationOpenAndClickFetcher.scala │ │ ├── OriginalTweetsFetcher.scala │ │ ├── ProfileClickFetcher.scala │ │ ├── ProfileVisitsFetcher.scala │ │ ├── RealGraphOonFetcher.scala │ │ ├── ReplyTweetsFetcher.scala │ │ ├── RetweetsFetcher.scala │ │ ├── SignalFilter.scala │ │ ├── TweetClickFetcher.scala │ │ ├── TweetFavoritesFetcher.scala │ │ ├── TweetSharesFetcher.scala │ │ ├── VideoTweetsPlayback50Fetcher.scala │ │ ├── VideoTweetsQualityViewFetcher.scala │ │ └── common/ │ │ ├── BUILD │ │ └── SGSUtils.scala │ └── thrift/ │ └── src/ │ └── main/ │ └── thrift/ │ ├── BUILD │ ├── client_identifier.thrift │ ├── service.thrift │ └── signal.thrift └── visibilitylib/ ├── BUILD ├── README.md └── src/ └── main/ ├── resources/ │ └── config/ │ ├── BUILD │ └── com/ │ └── twitter/ │ └── visibility/ │ └── decider.yml └── scala/ └── com/ └── twitter/ └── visibility/ ├── BUILD ├── VisibilityLibrary.scala ├── builder/ │ ├── BUILD │ ├── FeatureMapBuilder.scala │ ├── VerdictLogger.scala │ ├── VisibilityResult.scala │ ├── VisibilityResultBuilder.scala │ ├── common/ │ │ ├── BUILD │ │ └── MutedKeywordFeatures.scala │ ├── dms/ │ │ ├── BUILD │ │ ├── DmConversationFeatures.scala │ │ └── DmEventFeatures.scala │ ├── media/ │ │ ├── BUILD │ │ ├── MediaFeatures.scala │ │ └── MediaMetadataFeatures.scala │ ├── spaces/ │ │ ├── BUILD │ │ └── SpaceFeatures.scala │ ├── tweets/ │ │ ├── BUILD │ │ ├── BlenderContextFeatures.scala │ │ ├── CommunityNotificationFeatures.scala │ │ ├── CommunityTweetFeatures.scala │ │ ├── CommunityTweetFeaturesPartitioned.scala │ │ ├── CommunityTweetFeaturesV2.scala │ │ ├── ConversationControlFeatures.scala │ │ ├── EditTweetFeatures.scala │ │ ├── ExclusiveTweetFeatures.scala │ │ ├── FosnrPefetchedLabelsRelationshipFeatures.scala │ │ ├── FosnrRelationshipFeatures.scala │ │ ├── MisinformationPolicyFeatures.scala │ │ ├── ModerationFeatures.scala │ │ ├── SearchContextFeatures.scala │ │ ├── ToxicReplyFilterFeature.scala │ │ ├── TrustedFriendsFeatures.scala │ │ ├── TweetFeatures.scala │ │ ├── TweetIdFeatures.scala │ │ ├── TweetMediaMetadataFeatures.scala │ │ ├── TweetPerspectiveFeatures.scala │ │ ├── TweetVisibilityNudgeSourceWrapper.scala │ │ └── UnmentionNotificationFeatures.scala │ └── users/ │ ├── AuthorDeviceFeatures.scala │ ├── AuthorFeatures.scala │ ├── BUILD │ ├── QuotedTweetFeatures.scala │ ├── RelationshipFeatures.scala │ ├── RelationshipVerbHelpers.scala │ ├── SearchFeatures.scala │ ├── UserUnavailableFeatures.scala │ ├── ViewerAdvancedFilteringFeatures.scala │ ├── ViewerFeatures.scala │ ├── ViewerSearchSafetyFeatures.scala │ └── ViewerSensitiveMediaSettingsFeatures.scala ├── configapi/ │ ├── BUILD │ ├── ConfigBuilder.scala │ ├── VisibilityParams.scala │ ├── VisibilityRequestContext.scala │ ├── VisibilityRequestContextFactory.scala │ ├── configs/ │ │ ├── BUILD │ │ ├── DeciderKey.scala │ │ ├── ExperimentsHelper.scala │ │ ├── VisibilityDeciderGates.scala │ │ ├── VisibilityDeciders.scala │ │ ├── VisibilityExperimentsConfig.scala │ │ ├── VisibilityFeatureSwitches.scala │ │ └── overrides/ │ │ ├── BUILD │ │ └── VisibilityLibraryDeciderOverrides.scala │ └── params/ │ ├── BUILD │ ├── FSRuleParams.scala │ ├── GlobalParams.scala │ ├── LabelSourceParams.scala │ ├── RuleParams.scala │ ├── SafetyLevelParams.scala │ ├── TimelineConversationsDownrankingSpecificParams.scala │ ├── VisibilityExperiment.scala │ └── VisibilityExperiments.scala ├── engine/ │ ├── BUILD │ ├── DeciderableVisibilityRuleEngine.scala │ ├── VisibilityResultsMetricRecorder.scala │ ├── VisibilityRuleEngine.scala │ └── VisibilityRulePreprocessor.scala ├── features/ │ ├── AdvancedFilteringFeatures.scala │ ├── BUILD │ ├── Feature.scala │ ├── FeatureMap.scala │ └── Features.scala ├── generators/ │ ├── BUILD │ ├── CountryNameGenerator.scala │ ├── EpitaphToLocalizedMessage.scala │ ├── InterstitialReasonToLocalizedMessage.scala │ ├── LocalizedInterstitialGenerator.scala │ └── TombstoneGenerator.scala ├── interfaces/ │ ├── blender/ │ │ ├── BUILD │ │ ├── BlenderVisibilityLibrary.scala │ │ ├── BlenderVisibilityRequest.scala │ │ └── CombinedVisibilityResult.scala │ ├── cards/ │ │ ├── BUILD │ │ ├── CardVisibilityLibrary.scala │ │ ├── CardVisibilityLibraryParityTest.scala │ │ └── CardVisibilityRequest.scala │ ├── common/ │ │ ├── BUILD.bazel │ │ ├── blender/ │ │ │ ├── BUILD │ │ │ └── BlenderVFRequestContext.scala │ │ ├── search/ │ │ │ ├── BUILD │ │ │ └── SearchVFRequestContext.scala │ │ └── tweets/ │ │ ├── BUILD │ │ ├── StratoSafetyLabelFetcher.scala │ │ ├── StratoSafetyLabelMapFetcher.scala │ │ └── package.scala │ ├── conversations/ │ │ ├── AdAvoidanceLibrary.scala │ │ ├── BUILD │ │ ├── TimelineConversationsVisibilityLibrary.scala │ │ ├── TimelineConversationsVisibilityRequest.scala │ │ ├── TimelineConversationsVisibilityResponse.scala │ │ ├── Tombstone.scala │ │ ├── TombstoneVisibilityLibrary.scala │ │ └── package.scala │ ├── des/ │ │ ├── BUILD │ │ ├── DESRealtimeVisibilityLibrary.scala │ │ └── DESVisibilityLibrary.scala │ ├── dms/ │ │ ├── BUILD │ │ ├── DmConversationVisibilityLibrary.scala │ │ ├── DmConversationVisibilityRequest.scala │ │ ├── DmEventVisibilityLibrary.scala │ │ ├── DmEventVisibilityRequest.scala │ │ ├── DmVisibilityLibrary.scala │ │ └── package.scala │ ├── media/ │ │ ├── BUILD.bazel │ │ ├── MediaVisibilityLibrary.scala │ │ └── MediaVisibilityRequest.scala │ ├── notifications/ │ │ ├── BUILD │ │ ├── NotificationVFRequest.scala │ │ ├── NotificationsFilteringResponse.scala │ │ ├── NotificationsPlatformFilteringResponse.scala │ │ ├── NotificationsPlatformVisibilityLibrary.scala │ │ └── NotificationsVisibilityLibrary.scala │ ├── push_service/ │ │ ├── BUILD.bazel │ │ ├── PushServiceSafetyLabelMapFetcher.scala │ │ ├── PushServiceVisibilityLibrary.scala │ │ ├── PushServiceVisibilityLibraryParity.scala │ │ ├── PushServiceVisibilityLibraryUtil.scala │ │ ├── PushServiceVisibilityRequest.scala │ │ └── PushServiceVisibilityResponse.scala │ ├── search/ │ │ ├── BUILD │ │ ├── BatchSearchVisibilityRequest.scala │ │ ├── BatchSearchVisibilityResponse.scala │ │ ├── CombinedVisibilityResult.scala │ │ ├── SearchVisibilityLibrary.scala │ │ └── TweetContext.scala │ ├── spaces/ │ │ ├── BUILD │ │ ├── SpaceVisibilityLibrary.scala │ │ └── SpaceVisibilityRequest.scala │ ├── tweets/ │ │ ├── BUILD │ │ ├── DeletedTweetVisibilityLibrary.scala │ │ ├── QuotedTweetVisibilityLibrary.scala │ │ ├── TweetVisibilityLibrary.scala │ │ ├── TweetVisibilityLibraryParityTest.scala │ │ ├── TweetVisibilityRequest.scala │ │ ├── TweetypieContext.scala │ │ ├── UserUnavailableStateVisibilityLibrary.scala │ │ ├── UserUnavailableStateVisibilityRequest.scala │ │ └── enrichments/ │ │ ├── BUILD │ │ ├── ComplianceTweetNoticeEnrichment.scala │ │ ├── LimitedActionsPolicyEnrichment.scala │ │ └── TweetVisibilityNudgeEnrichment.scala │ └── users/ │ ├── BUILD │ └── UserVisibilityLibrary.scala ├── models/ │ ├── BUILD │ ├── CommunityTweet.scala │ ├── ContentId.scala │ ├── LabelSource.scala │ ├── MediaSafetyLabelType.scala │ ├── MisinformationPolicy.scala │ ├── MutedKeyword.scala │ ├── SafetyLabel.scala │ ├── SafetyLabelMetadata.scala │ ├── SafetyLabelType.scala │ ├── SafetyLevel.scala │ ├── SafetyLevelGroup.scala │ ├── SemanticCoreAnnotation.scala │ ├── SpaceSafetyLabelType.scala │ ├── TweetDeleteReason.scala │ ├── TweetModelMetadata.scala │ ├── TweetSafetyLabel.scala │ ├── UnitOfDiversion.scala │ ├── UserAge.scala │ ├── UserLabel.scala │ ├── UserSensitiveMediaSettings.scala │ ├── UserUnavailableStateEnum.scala │ ├── ViewerContext.scala │ ├── ViolationLevel.scala │ └── package.scala ├── rules/ │ ├── Action.scala │ ├── AdvancedFilteringRules.scala │ ├── BUILD │ ├── CardRules.scala │ ├── ComposableActions.scala │ ├── Condition.scala │ ├── DmConversationRules.scala │ ├── DmEventRules.scala │ ├── DmVisibilityPolicies.scala │ ├── DownrankingRules.scala │ ├── EvaluationContext.scala │ ├── ExperimentBase.scala │ ├── FailClosedException.scala │ ├── FollowerRelations.scala │ ├── ForEmergencyUseOnly.scala │ ├── FreedomOfSpeechNotReach.scala │ ├── InterstitialIf.scala │ ├── PublicInterestRules.scala │ ├── Rule.scala │ ├── RuleActionSourceBuilder.scala │ ├── RuleBase.scala │ ├── Rules.scala │ ├── SafeSearchRules.scala │ ├── SearchBlenderRules.scala │ ├── SensitiveMediaSettingsRules.scala │ ├── SpaceRules.scala │ ├── TombstoneIf.scala │ ├── ToxicityReplyFilterRules.scala │ ├── TweetLabelRules.scala │ ├── TweetRules.scala │ ├── UserLabelRules.scala │ ├── UserUnavailableStateTombstoneRules.scala │ ├── VisibilityPolicy.scala │ ├── generators/ │ │ ├── BUILD │ │ ├── RuleGenerator.scala │ │ ├── TweetRuleGenerator.scala │ │ └── TweetVisibilityPolicy.scala │ ├── package.scala │ ├── providers/ │ │ ├── BUILD │ │ ├── InjectedPolicyProvider.scala │ │ ├── PolicyProvider.scala │ │ └── ProvidedEvaluationContext.scala │ └── utils/ │ ├── BUILD │ └── ShimUtils.scala └── util/ ├── BUILD ├── DeciderUtil.scala ├── FeatureSwitchUtil.scala ├── LoggingUtil.scala └── NamingUtils.scala ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ .DS_Store ================================================ FILE: COPYING ================================================ GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software. A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public. The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version. An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU Affero General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Remote Network Interaction; Use with the GNU General Public License. Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements. You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see . ================================================ FILE: README.md ================================================ # X's Recommendation Algorithm X's Recommendation Algorithm is a set of services and jobs that are responsible for serving feeds of posts and other content across all X product surfaces (e.g. For You Timeline, Search, Explore, Notifications). For an introduction to how the algorithm works, please refer to our [engineering blog](https://blog.x.com/engineering/en_us/topics/open-source/2023/twitter-recommendation-algorithm). ## Architecture Product surfaces at X are built on a shared set of data, models, and software frameworks. The shared components included in this repository are listed below: | Type | Component | Description | |------------|------------|------------| | Data | [tweetypie](tweetypie/server/README.md) | Core service that handles the reading and writing of post data. | | | [unified-user-actions](unified_user_actions/README.md) | Real-time stream of user actions on X. | | | [user-signal-service](user-signal-service/README.md) | Centralized platform to retrieve explicit (e.g. likes, replies) and implicit (e.g. profile visits, tweet clicks) user signals. | | Model | [SimClusters](src/scala/com/twitter/simclusters_v2/README.md) | Community detection and sparse embeddings into those communities. | | | [TwHIN](https://github.com/twitter/the-algorithm-ml/blob/main/projects/twhin/README.md) | Dense knowledge graph embeddings for Users and Posts. | | | [trust-and-safety-models](trust_and_safety_models/README.md) | Models for detecting NSFW or abusive content. | | | [real-graph](src/scala/com/twitter/interaction_graph/README.md) | Model to predict the likelihood of an X User interacting with another User. | | | [tweepcred](src/scala/com/twitter/graph/batch/job/tweepcred/README) | Page-Rank algorithm for calculating X User reputation. | | | [recos-injector](recos-injector/README.md) | Streaming event processor for building input streams for [GraphJet](https://github.com/twitter/GraphJet) based services. | | | [graph-feature-service](graph-feature-service/README.md) | Serves graph features for a directed pair of users (e.g. how many of User A's following liked posts from User B). | | | [topic-social-proof](topic-social-proof/README.md) | Identifies topics related to individual posts. | | | [representation-scorer](representation-scorer/README.md) | Compute scores between pairs of entities (Users, Posts, etc.) using embedding similarity. | | Software framework | [navi](navi/README.md) | High performance, machine learning model serving written in Rust. | | | [product-mixer](product-mixer/README.md) | Software framework for building feeds of content. | | | [timelines-aggregation-framework](timelines/data_processing/ml_util/aggregation_framework/README.md) | Framework for generating aggregate features in batch or real time. | | | [representation-manager](representation-manager/README.md) | Service to retrieve embeddings (i.e. SimClusers and TwHIN). | | | [twml](twml/README.md) | Legacy machine learning framework built on TensorFlow v1. | The product surfaces currently included in this repository are the For You Timeline and Recommended Notifications. ### For You Timeline The diagram below illustrates how major services and jobs interconnect to construct a For You Timeline. ![](docs/system-diagram.png) The core components of the For You Timeline included in this repository are listed below: | Type | Component | Description | |------------|------------|------------| | Candidate Source | [search-index](src/java/com/twitter/search/README.md) | Find and rank In-Network posts. ~50% of posts come from this candidate source. | | | [tweet-mixer](tweet-mixer) | Coordination layer for fetching Out-of-Network tweet candidates from underlying compute services. | | | [user-tweet-entity-graph](src/scala/com/twitter/recos/user_tweet_entity_graph/README.md) (UTEG)| Maintains an in memory User to Post interaction graph, and finds candidates based on traversals of this graph. This is built on the [GraphJet](https://github.com/twitter/GraphJet) framework. Several other GraphJet based features and candidate sources are located [here](src/scala/com/twitter/recos). | | | [follow-recommendation-service](follow-recommendations-service/README.md) (FRS)| Provides Users with recommendations for accounts to follow, and posts from those accounts. | | Ranking | [light-ranker](src/python/twitter/deepbird/projects/timelines/scripts/models/earlybird/README.md) | Light Ranker model used by search index (Earlybird) to rank posts. | | | [heavy-ranker](https://github.com/twitter/the-algorithm-ml/blob/main/projects/home/recap/README.md) | Neural network for ranking candidate posts. One of the main signals used to select timeline posts post candidate sourcing. | | Post mixing & filtering | [home-mixer](home-mixer/README.md) | Main service used to construct and serve the Home Timeline. Built on [product-mixer](product-mixer/README.md). | | | [visibility-filters](visibilitylib/README.md) | Responsible for filtering X content to support legal compliance, improve product quality, increase user trust, protect revenue through the use of hard-filtering, visible product treatments, and coarse-grained downranking. | | | [timelineranker](timelineranker/README.md) | Legacy service which provides relevance-scored posts from the Earlybird Search Index and UTEG service. | ### Recommended Notifications The core components of Recommended Notifications included in this repository are listed below: | Type | Component | Description | |------------|------------|------------| | Service | [pushservice](pushservice/README.md) | Main recommendation service at X used to surface recommendations to our users via notifications. | Ranking | [pushservice-light-ranker](pushservice/src/main/python/models/light_ranking/README.md) | Light Ranker model used by pushservice to rank posts. Bridges candidate generation and heavy ranking by pre-selecting highly-relevant candidates from the initial huge candidate pool. | | | [pushservice-heavy-ranker](pushservice/src/main/python/models/heavy_ranking/README.md) | Multi-task learning model to predict the probabilities that the target users will open and engage with the sent notifications. | ## Build and test code We include Bazel BUILD files for most components, but not a top-level BUILD or WORKSPACE file. We plan to add a more complete build and test system in the future. ## Contributing We invite the community to submit GitHub issues and pull requests for suggestions on improving the recommendation algorithm. We are working on tools to manage these suggestions and sync changes to our internal repository. Any security concerns or issues should be routed to our official [bug bounty program](https://hackerone.com/x) through HackerOne. We hope to benefit from the collective intelligence and expertise of the global community in helping us identify issues and suggest improvements, ultimately leading to a better X. Read our blog on the open source initiative [here](https://blog.x.com/en_us/topics/company/2023/a-new-era-of-transparency-for-twitter). ================================================ FILE: RETREIVAL_SIGNALS.md ================================================ # Signals for Candidate Sources ## Overview The candidate sourcing stage within the Twitter Recommendation algorithm serves to significantly narrow down the item size from approximately 1 billion to just a few thousand. This process utilizes Twitter user behavior as the primary input for the algorithm. This document comprehensively enumerates all the signals during the candidate sourcing phase. | Signals | Description | | :-------------------- | :-------------------------------------------------------------------- | | Author Follow | The accounts which user explicit follows. | | Author Unfollow | The accounts which user recently unfollows. | | Author Mute | The accounts which user have muted. | | Author Block | The accounts which user have blocked | | Tweet Favorite | The tweets which user clicked the like botton. | | Tweet Unfavorite | The tweets which user clicked the unlike botton. | | Retweet | The tweets which user retweeted | | Quote Tweet | The tweets which user retweeted with comments. | | Tweet Reply | The tweets which user replied. | | Tweet Share | The tweets which user clicked the share botton. | | Tweet Bookmark | The tweets which user clicked the bookmark botton. | | Tweet Click | The tweets which user clicked and viewed the tweet detail page. | | Tweet Video Watch | The video tweets which user watched certain seconds or percentage. | | Tweet Don't like | The tweets which user clicked "Not interested in this tweet" botton. | | Tweet Report | The tweets which user clicked "Report Tweet" botton. | | Notification Open | The push notification tweets which user opened. | | Ntab click | The tweets which user click on the Notifications page. | | User AddressBook | The author accounts identifiers of the user's addressbook. | ## Usage Details Twitter uses these user signals as training labels and/or ML features in the each candidate sourcing algorithms. The following tables shows how they are used in the each components. | Signals | USS | SimClusters | TwHin | UTEG | FRS | Light Ranking | | :-------------------- | :----------------- | :----------------- | :----------------- | :----------------- | :----------------- | :----------------- | | Author Follow | Features | Features / Labels | Features / Labels | Features | Features / Labels | N/A | | Author Unfollow | Features | N/A | N/A | N/A | N/A | N/A | | Author Mute | Features | N/A | N/A | N/A | Features | N/A | | Author Block | Features | N/A | N/A | N/A | Features | N/A | | Tweet Favorite | Features | Features | Features / Labels | Features | Features / Labels | Features / Labels | | Tweet Unfavorite | Features | Features | N/A | N/A | N/A | N/A | | Retweet | Features | N/A | Features / Labels | Features | Features / Labels | Features / Labels | | Quote Tweet | Features | N/A | Features / Labels | Features | Features / Labels | Features / Labels | | Tweet Reply | Features | N/A | Features | Features | Features / Labels | Features | | Tweet Share | Features | N/A | N/A | N/A | Features | N/A | | Tweet Bookmark | Features | N/A | N/A | N/A | N/A | N/A | | Tweet Click | Features | N/A | N/A | N/A | Features | Labels | | Tweet Video Watch | Features | Features | N/A | N/A | N/A | Labels | | Tweet Don't like | Features | N/A | N/A | N/A | N/A | N/A | | Tweet Report | Features | N/A | N/A | N/A | N/A | N/A | | Notification Open | Features | Features | Features | N/A | Features | N/A | | Ntab click | Features | Features | Features | N/A | Features | N/A | | User AddressBook | N/A | N/A | N/A | N/A | Features | N/A | ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/BUILD ================================================ target( name = "faiss", dependencies = [ "ann/src/main/java/com/twitter/ann/faiss/swig:swig-artifactory", ], ) java_library( name = "swig-native-utils", sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [], ) ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/NativeUtils.java ================================================ package com.twitter.ann.faiss; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.StandardCopyOption; import java.util.Locale; public final class NativeUtils { private static final int MIN_PREFIX_LENGTH = 3; public static final String NATIVE_FOLDER_PATH_PREFIX = "nativeutils"; public static File temporaryDir; private NativeUtils() { } private static File unpackLibraryFromJarInternal(String path) throws IOException { if (null == path || !path.startsWith("/")) { throw new IllegalArgumentException("The path has to be absolute (start with '/')."); } String[] parts = path.split("/"); String filename = (parts.length > 1) ? parts[parts.length - 1] : null; if (filename == null || filename.length() < MIN_PREFIX_LENGTH) { throw new IllegalArgumentException("The filename has to be at least 3 characters long."); } if (temporaryDir == null) { temporaryDir = createTempDirectory(NATIVE_FOLDER_PATH_PREFIX); temporaryDir.deleteOnExit(); } File temp = new File(temporaryDir, filename); try (InputStream is = NativeUtils.class.getResourceAsStream(path)) { Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { temp.delete(); throw e; } catch (NullPointerException e) { temp.delete(); throw new FileNotFoundException("File " + path + " was not found inside JAR."); } return temp; } /** * Unpack library from JAR into temporary path * * @param path The path of file inside JAR as absolute path (beginning with * '/'), e.g. /package/File.ext * @throws IOException If temporary file creation or read/write * operation fails * @throws IllegalArgumentException If source file (param path) does not exist * @throws IllegalArgumentException If the path is not absolute or if the * filename is shorter than three characters * (restriction of * {@link File#createTempFile(java.lang.String, java.lang.String)}). * @throws FileNotFoundException If the file could not be found inside the * JAR. */ public static void unpackLibraryFromJar(String path) throws IOException { unpackLibraryFromJarInternal(path); } /** * Loads library from current JAR archive *

* The file from JAR is copied into system temporary directory and then loaded. * The temporary file is deleted after * exiting. * Method uses String as filename because the pathname is "abstract", not * system-dependent. * * @param path The path of file inside JAR as absolute path (beginning with * '/'), e.g. /package/File.ext * @throws IOException If temporary file creation or read/write * operation fails * @throws IllegalArgumentException If source file (param path) does not exist * @throws IllegalArgumentException If the path is not absolute or if the * filename is shorter than three characters * (restriction of * {@link File#createTempFile(java.lang.String, java.lang.String)}). * @throws FileNotFoundException If the file could not be found inside the * JAR. */ public static void loadLibraryFromJar(String path) throws IOException { File temp = unpackLibraryFromJarInternal(path); try (InputStream is = NativeUtils.class.getResourceAsStream(path)) { Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { temp.delete(); throw e; } catch (NullPointerException e) { temp.delete(); throw new FileNotFoundException("File " + path + " was not found inside JAR."); } try { System.load(temp.getAbsolutePath()); } finally { temp.deleteOnExit(); } } private static File createTempDirectory(String prefix) throws IOException { String tempDir = System.getProperty("java.io.tmpdir"); File generatedDir = new File(tempDir, prefix + System.nanoTime()); if (!generatedDir.mkdir()) { throw new IOException("Failed to create temp directory " + generatedDir.getName()); } return generatedDir; } public enum OSType { Windows, MacOS, Linux, Other } protected static OSType detectedOS; /** * detect the operating system from the os.name System property and cache * the result * * @returns - the operating system detected */ public static OSType getOperatingSystemType() { if (detectedOS == null) { String osname = System.getProperty("os.name", "generic").toLowerCase(Locale.ENGLISH); if ((osname.contains("mac")) || (osname.contains("darwin"))) { detectedOS = OSType.MacOS; } else if (osname.contains("win")) { detectedOS = OSType.Windows; } else if (osname.contains("nux")) { detectedOS = OSType.Linux; } else { detectedOS = OSType.Other; } } return detectedOS; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/AlignedTableFloat32.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class AlignedTableFloat32 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected AlignedTableFloat32(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(AlignedTableFloat32 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_AlignedTableFloat32(swigCPtr); } swigCPtr = 0; } } public void setTab(SWIGTYPE_p_faiss__AlignedTableTightAllocT_float_32_t value) { swigfaissJNI.AlignedTableFloat32_tab_set(swigCPtr, this, SWIGTYPE_p_faiss__AlignedTableTightAllocT_float_32_t.getCPtr(value)); } public SWIGTYPE_p_faiss__AlignedTableTightAllocT_float_32_t getTab() { long cPtr = swigfaissJNI.AlignedTableFloat32_tab_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_faiss__AlignedTableTightAllocT_float_32_t(cPtr, false); } public void setNumel(long value) { swigfaissJNI.AlignedTableFloat32_numel_set(swigCPtr, this, value); } public long getNumel() { return swigfaissJNI.AlignedTableFloat32_numel_get(swigCPtr, this); } public static long round_capacity(long n) { return swigfaissJNI.AlignedTableFloat32_round_capacity(n); } public AlignedTableFloat32() { this(swigfaissJNI.new_AlignedTableFloat32__SWIG_0(), true); } public AlignedTableFloat32(long n) { this(swigfaissJNI.new_AlignedTableFloat32__SWIG_1(n), true); } public long itemsize() { return swigfaissJNI.AlignedTableFloat32_itemsize(swigCPtr, this); } public void resize(long n) { swigfaissJNI.AlignedTableFloat32_resize(swigCPtr, this, n); } public void clear() { swigfaissJNI.AlignedTableFloat32_clear(swigCPtr, this); } public long size() { return swigfaissJNI.AlignedTableFloat32_size(swigCPtr, this); } public long nbytes() { return swigfaissJNI.AlignedTableFloat32_nbytes(swigCPtr, this); } public SWIGTYPE_p_float get() { long cPtr = swigfaissJNI.AlignedTableFloat32_get__SWIG_0(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } public SWIGTYPE_p_float data() { long cPtr = swigfaissJNI.AlignedTableFloat32_data__SWIG_0(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/AlignedTableUint16.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class AlignedTableUint16 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected AlignedTableUint16(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(AlignedTableUint16 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_AlignedTableUint16(swigCPtr); } swigCPtr = 0; } } public void setTab(SWIGTYPE_p_faiss__AlignedTableTightAllocT_uint16_t_32_t value) { swigfaissJNI.AlignedTableUint16_tab_set(swigCPtr, this, SWIGTYPE_p_faiss__AlignedTableTightAllocT_uint16_t_32_t.getCPtr(value)); } public SWIGTYPE_p_faiss__AlignedTableTightAllocT_uint16_t_32_t getTab() { long cPtr = swigfaissJNI.AlignedTableUint16_tab_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_faiss__AlignedTableTightAllocT_uint16_t_32_t(cPtr, false); } public void setNumel(long value) { swigfaissJNI.AlignedTableUint16_numel_set(swigCPtr, this, value); } public long getNumel() { return swigfaissJNI.AlignedTableUint16_numel_get(swigCPtr, this); } public static long round_capacity(long n) { return swigfaissJNI.AlignedTableUint16_round_capacity(n); } public AlignedTableUint16() { this(swigfaissJNI.new_AlignedTableUint16__SWIG_0(), true); } public AlignedTableUint16(long n) { this(swigfaissJNI.new_AlignedTableUint16__SWIG_1(n), true); } public long itemsize() { return swigfaissJNI.AlignedTableUint16_itemsize(swigCPtr, this); } public void resize(long n) { swigfaissJNI.AlignedTableUint16_resize(swigCPtr, this, n); } public void clear() { swigfaissJNI.AlignedTableUint16_clear(swigCPtr, this); } public long size() { return swigfaissJNI.AlignedTableUint16_size(swigCPtr, this); } public long nbytes() { return swigfaissJNI.AlignedTableUint16_nbytes(swigCPtr, this); } public SWIGTYPE_p_uint16_t get() { long cPtr = swigfaissJNI.AlignedTableUint16_get__SWIG_0(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_uint16_t(cPtr, false); } public SWIGTYPE_p_uint16_t data() { long cPtr = swigfaissJNI.AlignedTableUint16_data__SWIG_0(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_uint16_t(cPtr, false); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/AlignedTableUint8.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class AlignedTableUint8 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected AlignedTableUint8(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(AlignedTableUint8 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_AlignedTableUint8(swigCPtr); } swigCPtr = 0; } } public void setTab(SWIGTYPE_p_faiss__AlignedTableTightAllocT_unsigned_char_32_t value) { swigfaissJNI.AlignedTableUint8_tab_set(swigCPtr, this, SWIGTYPE_p_faiss__AlignedTableTightAllocT_unsigned_char_32_t.getCPtr(value)); } public SWIGTYPE_p_faiss__AlignedTableTightAllocT_unsigned_char_32_t getTab() { long cPtr = swigfaissJNI.AlignedTableUint8_tab_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_faiss__AlignedTableTightAllocT_unsigned_char_32_t(cPtr, false); } public void setNumel(long value) { swigfaissJNI.AlignedTableUint8_numel_set(swigCPtr, this, value); } public long getNumel() { return swigfaissJNI.AlignedTableUint8_numel_get(swigCPtr, this); } public static long round_capacity(long n) { return swigfaissJNI.AlignedTableUint8_round_capacity(n); } public AlignedTableUint8() { this(swigfaissJNI.new_AlignedTableUint8__SWIG_0(), true); } public AlignedTableUint8(long n) { this(swigfaissJNI.new_AlignedTableUint8__SWIG_1(n), true); } public long itemsize() { return swigfaissJNI.AlignedTableUint8_itemsize(swigCPtr, this); } public void resize(long n) { swigfaissJNI.AlignedTableUint8_resize(swigCPtr, this, n); } public void clear() { swigfaissJNI.AlignedTableUint8_clear(swigCPtr, this); } public long size() { return swigfaissJNI.AlignedTableUint8_size(swigCPtr, this); } public long nbytes() { return swigfaissJNI.AlignedTableUint8_nbytes(swigCPtr, this); } public SWIGTYPE_p_unsigned_char get() { long cPtr = swigfaissJNI.AlignedTableUint8_get__SWIG_0(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public SWIGTYPE_p_unsigned_char data() { long cPtr = swigfaissJNI.AlignedTableUint8_data__SWIG_0(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/ArrayInvertedLists.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class ArrayInvertedLists extends InvertedLists { private transient long swigCPtr; protected ArrayInvertedLists(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.ArrayInvertedLists_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(ArrayInvertedLists obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_ArrayInvertedLists(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setCodes(ByteVectorVector value) { swigfaissJNI.ArrayInvertedLists_codes_set(swigCPtr, this, ByteVectorVector.getCPtr(value), value); } public ByteVectorVector getCodes() { long cPtr = swigfaissJNI.ArrayInvertedLists_codes_get(swigCPtr, this); return (cPtr == 0) ? null : new ByteVectorVector(cPtr, false); } public void setIds(SWIGTYPE_p_std__vectorT_std__vectorT_int64_t_t_t value) { swigfaissJNI.ArrayInvertedLists_ids_set(swigCPtr, this, SWIGTYPE_p_std__vectorT_std__vectorT_int64_t_t_t.getCPtr(value)); } public SWIGTYPE_p_std__vectorT_std__vectorT_int64_t_t_t getIds() { long cPtr = swigfaissJNI.ArrayInvertedLists_ids_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_std__vectorT_int64_t_t_t(cPtr, false); } public ArrayInvertedLists(long nlist, long code_size) { this(swigfaissJNI.new_ArrayInvertedLists(nlist, code_size), true); } public long list_size(long list_no) { return swigfaissJNI.ArrayInvertedLists_list_size(swigCPtr, this, list_no); } public SWIGTYPE_p_unsigned_char get_codes(long list_no) { long cPtr = swigfaissJNI.ArrayInvertedLists_get_codes(swigCPtr, this, list_no); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public LongVector get_ids(long list_no) { return new LongVector(swigfaissJNI.ArrayInvertedLists_get_ids(swigCPtr, this, list_no), false); } public long add_entries(long list_no, long n_entry, LongVector ids, SWIGTYPE_p_unsigned_char code) { return swigfaissJNI.ArrayInvertedLists_add_entries(swigCPtr, this, list_no, n_entry, SWIGTYPE_p_long_long.getCPtr(ids.data()), ids, SWIGTYPE_p_unsigned_char.getCPtr(code)); } public void update_entries(long list_no, long offset, long n_entry, LongVector ids, SWIGTYPE_p_unsigned_char code) { swigfaissJNI.ArrayInvertedLists_update_entries(swigCPtr, this, list_no, offset, n_entry, SWIGTYPE_p_long_long.getCPtr(ids.data()), ids, SWIGTYPE_p_unsigned_char.getCPtr(code)); } public void resize(long list_no, long new_size) { swigfaissJNI.ArrayInvertedLists_resize(swigCPtr, this, list_no, new_size); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/AutoTuneCriterion.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class AutoTuneCriterion { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected AutoTuneCriterion(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(AutoTuneCriterion obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_AutoTuneCriterion(swigCPtr); } swigCPtr = 0; } } public void setNq(long value) { swigfaissJNI.AutoTuneCriterion_nq_set(swigCPtr, this, value); } public long getNq() { return swigfaissJNI.AutoTuneCriterion_nq_get(swigCPtr, this); } public void setNnn(long value) { swigfaissJNI.AutoTuneCriterion_nnn_set(swigCPtr, this, value); } public long getNnn() { return swigfaissJNI.AutoTuneCriterion_nnn_get(swigCPtr, this); } public void setGt_nnn(long value) { swigfaissJNI.AutoTuneCriterion_gt_nnn_set(swigCPtr, this, value); } public long getGt_nnn() { return swigfaissJNI.AutoTuneCriterion_gt_nnn_get(swigCPtr, this); } public void setGt_D(FloatVector value) { swigfaissJNI.AutoTuneCriterion_gt_D_set(swigCPtr, this, FloatVector.getCPtr(value), value); } public FloatVector getGt_D() { long cPtr = swigfaissJNI.AutoTuneCriterion_gt_D_get(swigCPtr, this); return (cPtr == 0) ? null : new FloatVector(cPtr, false); } public void setGt_I(SWIGTYPE_p_std__vectorT_int64_t_t value) { swigfaissJNI.AutoTuneCriterion_gt_I_set(swigCPtr, this, SWIGTYPE_p_std__vectorT_int64_t_t.getCPtr(value)); } public SWIGTYPE_p_std__vectorT_int64_t_t getGt_I() { long cPtr = swigfaissJNI.AutoTuneCriterion_gt_I_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_int64_t_t(cPtr, false); } public void set_groundtruth(int gt_nnn, SWIGTYPE_p_float gt_D_in, LongVector gt_I_in) { swigfaissJNI.AutoTuneCriterion_set_groundtruth(swigCPtr, this, gt_nnn, SWIGTYPE_p_float.getCPtr(gt_D_in), SWIGTYPE_p_long_long.getCPtr(gt_I_in.data()), gt_I_in); } public double evaluate(SWIGTYPE_p_float D, LongVector I) { return swigfaissJNI.AutoTuneCriterion_evaluate(swigCPtr, this, SWIGTYPE_p_float.getCPtr(D), SWIGTYPE_p_long_long.getCPtr(I.data()), I); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/BUILD ================================================ java_library( name = "swig-local", sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = [ "bazel-compatible", "bazel-only", ], dependencies = [ "ann/src/main/java/com/twitter/ann/faiss:swig-native-utils", "ann/src/main/java/com/twitter/ann/faiss/swig/resources", ], ) java_library( name = "swig-artifactory", sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/ann/faiss/swig:resources", "ann/src/main/java/com/twitter/ann/faiss:swig-native-utils", ], ) ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/BitstringReader.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class BitstringReader { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected BitstringReader(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(BitstringReader obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_BitstringReader(swigCPtr); } swigCPtr = 0; } } public void setCode(SWIGTYPE_p_unsigned_char value) { swigfaissJNI.BitstringReader_code_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(value)); } public SWIGTYPE_p_unsigned_char getCode() { long cPtr = swigfaissJNI.BitstringReader_code_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public void setCode_size(long value) { swigfaissJNI.BitstringReader_code_size_set(swigCPtr, this, value); } public long getCode_size() { return swigfaissJNI.BitstringReader_code_size_get(swigCPtr, this); } public void setI(long value) { swigfaissJNI.BitstringReader_i_set(swigCPtr, this, value); } public long getI() { return swigfaissJNI.BitstringReader_i_get(swigCPtr, this); } public BitstringReader(SWIGTYPE_p_unsigned_char code, long code_size) { this(swigfaissJNI.new_BitstringReader(SWIGTYPE_p_unsigned_char.getCPtr(code), code_size), true); } public long read(int nbit) { return swigfaissJNI.BitstringReader_read(swigCPtr, this, nbit); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/BitstringWriter.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class BitstringWriter { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected BitstringWriter(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(BitstringWriter obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_BitstringWriter(swigCPtr); } swigCPtr = 0; } } public void setCode(SWIGTYPE_p_unsigned_char value) { swigfaissJNI.BitstringWriter_code_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(value)); } public SWIGTYPE_p_unsigned_char getCode() { long cPtr = swigfaissJNI.BitstringWriter_code_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public void setCode_size(long value) { swigfaissJNI.BitstringWriter_code_size_set(swigCPtr, this, value); } public long getCode_size() { return swigfaissJNI.BitstringWriter_code_size_get(swigCPtr, this); } public void setI(long value) { swigfaissJNI.BitstringWriter_i_set(swigCPtr, this, value); } public long getI() { return swigfaissJNI.BitstringWriter_i_get(swigCPtr, this); } public BitstringWriter(SWIGTYPE_p_unsigned_char code, long code_size) { this(swigfaissJNI.new_BitstringWriter(SWIGTYPE_p_unsigned_char.getCPtr(code), code_size), true); } public void write(long x, int nbit) { swigfaissJNI.BitstringWriter_write(swigCPtr, this, x, nbit); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/BufferList.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class BufferList { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected BufferList(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(BufferList obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_BufferList(swigCPtr); } swigCPtr = 0; } } public void setBuffer_size(long value) { swigfaissJNI.BufferList_buffer_size_set(swigCPtr, this, value); } public long getBuffer_size() { return swigfaissJNI.BufferList_buffer_size_get(swigCPtr, this); } public void setBuffers(SWIGTYPE_p_std__vectorT_faiss__BufferList__Buffer_t value) { swigfaissJNI.BufferList_buffers_set(swigCPtr, this, SWIGTYPE_p_std__vectorT_faiss__BufferList__Buffer_t.getCPtr(value)); } public SWIGTYPE_p_std__vectorT_faiss__BufferList__Buffer_t getBuffers() { long cPtr = swigfaissJNI.BufferList_buffers_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_faiss__BufferList__Buffer_t(cPtr, false); } public void setWp(long value) { swigfaissJNI.BufferList_wp_set(swigCPtr, this, value); } public long getWp() { return swigfaissJNI.BufferList_wp_get(swigCPtr, this); } public BufferList(long buffer_size) { this(swigfaissJNI.new_BufferList(buffer_size), true); } public void append_buffer() { swigfaissJNI.BufferList_append_buffer(swigCPtr, this); } public void add(long id, float dis) { swigfaissJNI.BufferList_add(swigCPtr, this, id, dis); } public void copy_range(long ofs, long n, LongVector dest_ids, SWIGTYPE_p_float dest_dis) { swigfaissJNI.BufferList_copy_range(swigCPtr, this, ofs, n, SWIGTYPE_p_long_long.getCPtr(dest_ids.data()), dest_ids, SWIGTYPE_p_float.getCPtr(dest_dis)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/ByteVector.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class ByteVector { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected ByteVector(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(ByteVector obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_ByteVector(swigCPtr); } swigCPtr = 0; } } public ByteVector() { this(swigfaissJNI.new_ByteVector(), true); } public void push_back(short arg0) { swigfaissJNI.ByteVector_push_back(swigCPtr, this, arg0); } public void clear() { swigfaissJNI.ByteVector_clear(swigCPtr, this); } public SWIGTYPE_p_unsigned_char data() { long cPtr = swigfaissJNI.ByteVector_data(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public long size() { return swigfaissJNI.ByteVector_size(swigCPtr, this); } public short at(long n) { return swigfaissJNI.ByteVector_at(swigCPtr, this, n); } public void resize(long n) { swigfaissJNI.ByteVector_resize(swigCPtr, this, n); } public void reserve(long n) { swigfaissJNI.ByteVector_reserve(swigCPtr, this, n); } public void swap(ByteVector other) { swigfaissJNI.ByteVector_swap(swigCPtr, this, ByteVector.getCPtr(other), other); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/ByteVectorVector.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class ByteVectorVector { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected ByteVectorVector(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(ByteVectorVector obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_ByteVectorVector(swigCPtr); } swigCPtr = 0; } } public ByteVectorVector() { this(swigfaissJNI.new_ByteVectorVector(), true); } public void push_back(ByteVector arg0) { swigfaissJNI.ByteVectorVector_push_back(swigCPtr, this, ByteVector.getCPtr(arg0), arg0); } public void clear() { swigfaissJNI.ByteVectorVector_clear(swigCPtr, this); } public ByteVector data() { long cPtr = swigfaissJNI.ByteVectorVector_data(swigCPtr, this); return (cPtr == 0) ? null : new ByteVector(cPtr, false); } public long size() { return swigfaissJNI.ByteVectorVector_size(swigCPtr, this); } public ByteVector at(long n) { return new ByteVector(swigfaissJNI.ByteVectorVector_at(swigCPtr, this, n), true); } public void resize(long n) { swigfaissJNI.ByteVectorVector_resize(swigCPtr, this, n); } public void reserve(long n) { swigfaissJNI.ByteVectorVector_reserve(swigCPtr, this, n); } public void swap(ByteVectorVector other) { swigfaissJNI.ByteVectorVector_swap(swigCPtr, this, ByteVectorVector.getCPtr(other), other); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/CenteringTransform.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class CenteringTransform extends VectorTransform { private transient long swigCPtr; protected CenteringTransform(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.CenteringTransform_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(CenteringTransform obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_CenteringTransform(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setMean(FloatVector value) { swigfaissJNI.CenteringTransform_mean_set(swigCPtr, this, FloatVector.getCPtr(value), value); } public FloatVector getMean() { long cPtr = swigfaissJNI.CenteringTransform_mean_get(swigCPtr, this); return (cPtr == 0) ? null : new FloatVector(cPtr, false); } public CenteringTransform(int d) { this(swigfaissJNI.new_CenteringTransform__SWIG_0(d), true); } public CenteringTransform() { this(swigfaissJNI.new_CenteringTransform__SWIG_1(), true); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.CenteringTransform_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void apply_noalloc(long n, SWIGTYPE_p_float x, SWIGTYPE_p_float xt) { swigfaissJNI.CenteringTransform_apply_noalloc(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_float.getCPtr(xt)); } public void reverse_transform(long n, SWIGTYPE_p_float xt, SWIGTYPE_p_float x) { swigfaissJNI.CenteringTransform_reverse_transform(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(xt), SWIGTYPE_p_float.getCPtr(x)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/CharVector.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class CharVector { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected CharVector(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(CharVector obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_CharVector(swigCPtr); } swigCPtr = 0; } } public CharVector() { this(swigfaissJNI.new_CharVector(), true); } public void push_back(char arg0) { swigfaissJNI.CharVector_push_back(swigCPtr, this, arg0); } public void clear() { swigfaissJNI.CharVector_clear(swigCPtr, this); } public String data() { return swigfaissJNI.CharVector_data(swigCPtr, this); } public long size() { return swigfaissJNI.CharVector_size(swigCPtr, this); } public char at(long n) { return swigfaissJNI.CharVector_at(swigCPtr, this, n); } public void resize(long n) { swigfaissJNI.CharVector_resize(swigCPtr, this, n); } public void reserve(long n) { swigfaissJNI.CharVector_reserve(swigCPtr, this, n); } public void swap(CharVector other) { swigfaissJNI.CharVector_swap(swigCPtr, this, CharVector.getCPtr(other), other); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/Clustering.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class Clustering extends ClusteringParameters { private transient long swigCPtr; protected Clustering(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.Clustering_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(Clustering obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_Clustering(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setD(long value) { swigfaissJNI.Clustering_d_set(swigCPtr, this, value); } public long getD() { return swigfaissJNI.Clustering_d_get(swigCPtr, this); } public void setK(long value) { swigfaissJNI.Clustering_k_set(swigCPtr, this, value); } public long getK() { return swigfaissJNI.Clustering_k_get(swigCPtr, this); } public void setCentroids(FloatVector value) { swigfaissJNI.Clustering_centroids_set(swigCPtr, this, FloatVector.getCPtr(value), value); } public FloatVector getCentroids() { long cPtr = swigfaissJNI.Clustering_centroids_get(swigCPtr, this); return (cPtr == 0) ? null : new FloatVector(cPtr, false); } public void setIteration_stats(SWIGTYPE_p_std__vectorT_faiss__ClusteringIterationStats_t value) { swigfaissJNI.Clustering_iteration_stats_set(swigCPtr, this, SWIGTYPE_p_std__vectorT_faiss__ClusteringIterationStats_t.getCPtr(value)); } public SWIGTYPE_p_std__vectorT_faiss__ClusteringIterationStats_t getIteration_stats() { long cPtr = swigfaissJNI.Clustering_iteration_stats_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_faiss__ClusteringIterationStats_t(cPtr, false); } public Clustering(int d, int k) { this(swigfaissJNI.new_Clustering__SWIG_0(d, k), true); } public Clustering(int d, int k, ClusteringParameters cp) { this(swigfaissJNI.new_Clustering__SWIG_1(d, k, ClusteringParameters.getCPtr(cp), cp), true); } public void train(long n, SWIGTYPE_p_float x, Index index, SWIGTYPE_p_float x_weights) { swigfaissJNI.Clustering_train__SWIG_0(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), Index.getCPtr(index), index, SWIGTYPE_p_float.getCPtr(x_weights)); } public void train(long n, SWIGTYPE_p_float x, Index index) { swigfaissJNI.Clustering_train__SWIG_1(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), Index.getCPtr(index), index); } public void train_encoded(long nx, SWIGTYPE_p_unsigned_char x_in, Index codec, Index index, SWIGTYPE_p_float weights) { swigfaissJNI.Clustering_train_encoded__SWIG_0(swigCPtr, this, nx, SWIGTYPE_p_unsigned_char.getCPtr(x_in), Index.getCPtr(codec), codec, Index.getCPtr(index), index, SWIGTYPE_p_float.getCPtr(weights)); } public void train_encoded(long nx, SWIGTYPE_p_unsigned_char x_in, Index codec, Index index) { swigfaissJNI.Clustering_train_encoded__SWIG_1(swigCPtr, this, nx, SWIGTYPE_p_unsigned_char.getCPtr(x_in), Index.getCPtr(codec), codec, Index.getCPtr(index), index); } public void post_process_centroids() { swigfaissJNI.Clustering_post_process_centroids(swigCPtr, this); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/Clustering1D.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class Clustering1D extends Clustering { private transient long swigCPtr; protected Clustering1D(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.Clustering1D_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(Clustering1D obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_Clustering1D(swigCPtr); } swigCPtr = 0; } super.delete(); } public Clustering1D(int k) { this(swigfaissJNI.new_Clustering1D__SWIG_0(k), true); } public Clustering1D(int k, ClusteringParameters cp) { this(swigfaissJNI.new_Clustering1D__SWIG_1(k, ClusteringParameters.getCPtr(cp), cp), true); } public void train_exact(long n, SWIGTYPE_p_float x) { swigfaissJNI.Clustering1D_train_exact(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/ClusteringIterationStats.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class ClusteringIterationStats { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected ClusteringIterationStats(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(ClusteringIterationStats obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_ClusteringIterationStats(swigCPtr); } swigCPtr = 0; } } public void setObj(float value) { swigfaissJNI.ClusteringIterationStats_obj_set(swigCPtr, this, value); } public float getObj() { return swigfaissJNI.ClusteringIterationStats_obj_get(swigCPtr, this); } public void setTime(double value) { swigfaissJNI.ClusteringIterationStats_time_set(swigCPtr, this, value); } public double getTime() { return swigfaissJNI.ClusteringIterationStats_time_get(swigCPtr, this); } public void setTime_search(double value) { swigfaissJNI.ClusteringIterationStats_time_search_set(swigCPtr, this, value); } public double getTime_search() { return swigfaissJNI.ClusteringIterationStats_time_search_get(swigCPtr, this); } public void setImbalance_factor(double value) { swigfaissJNI.ClusteringIterationStats_imbalance_factor_set(swigCPtr, this, value); } public double getImbalance_factor() { return swigfaissJNI.ClusteringIterationStats_imbalance_factor_get(swigCPtr, this); } public void setNsplit(int value) { swigfaissJNI.ClusteringIterationStats_nsplit_set(swigCPtr, this, value); } public int getNsplit() { return swigfaissJNI.ClusteringIterationStats_nsplit_get(swigCPtr, this); } public ClusteringIterationStats() { this(swigfaissJNI.new_ClusteringIterationStats(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/ClusteringParameters.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class ClusteringParameters { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected ClusteringParameters(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(ClusteringParameters obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_ClusteringParameters(swigCPtr); } swigCPtr = 0; } } public void setNiter(int value) { swigfaissJNI.ClusteringParameters_niter_set(swigCPtr, this, value); } public int getNiter() { return swigfaissJNI.ClusteringParameters_niter_get(swigCPtr, this); } public void setNredo(int value) { swigfaissJNI.ClusteringParameters_nredo_set(swigCPtr, this, value); } public int getNredo() { return swigfaissJNI.ClusteringParameters_nredo_get(swigCPtr, this); } public void setVerbose(boolean value) { swigfaissJNI.ClusteringParameters_verbose_set(swigCPtr, this, value); } public boolean getVerbose() { return swigfaissJNI.ClusteringParameters_verbose_get(swigCPtr, this); } public void setSpherical(boolean value) { swigfaissJNI.ClusteringParameters_spherical_set(swigCPtr, this, value); } public boolean getSpherical() { return swigfaissJNI.ClusteringParameters_spherical_get(swigCPtr, this); } public void setInt_centroids(boolean value) { swigfaissJNI.ClusteringParameters_int_centroids_set(swigCPtr, this, value); } public boolean getInt_centroids() { return swigfaissJNI.ClusteringParameters_int_centroids_get(swigCPtr, this); } public void setUpdate_index(boolean value) { swigfaissJNI.ClusteringParameters_update_index_set(swigCPtr, this, value); } public boolean getUpdate_index() { return swigfaissJNI.ClusteringParameters_update_index_get(swigCPtr, this); } public void setFrozen_centroids(boolean value) { swigfaissJNI.ClusteringParameters_frozen_centroids_set(swigCPtr, this, value); } public boolean getFrozen_centroids() { return swigfaissJNI.ClusteringParameters_frozen_centroids_get(swigCPtr, this); } public void setMin_points_per_centroid(int value) { swigfaissJNI.ClusteringParameters_min_points_per_centroid_set(swigCPtr, this, value); } public int getMin_points_per_centroid() { return swigfaissJNI.ClusteringParameters_min_points_per_centroid_get(swigCPtr, this); } public void setMax_points_per_centroid(int value) { swigfaissJNI.ClusteringParameters_max_points_per_centroid_set(swigCPtr, this, value); } public int getMax_points_per_centroid() { return swigfaissJNI.ClusteringParameters_max_points_per_centroid_get(swigCPtr, this); } public void setSeed(int value) { swigfaissJNI.ClusteringParameters_seed_set(swigCPtr, this, value); } public int getSeed() { return swigfaissJNI.ClusteringParameters_seed_get(swigCPtr, this); } public void setDecode_block_size(long value) { swigfaissJNI.ClusteringParameters_decode_block_size_set(swigCPtr, this, value); } public long getDecode_block_size() { return swigfaissJNI.ClusteringParameters_decode_block_size_get(swigCPtr, this); } public ClusteringParameters() { this(swigfaissJNI.new_ClusteringParameters(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/DistanceComputer.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class DistanceComputer { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected DistanceComputer(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(DistanceComputer obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_DistanceComputer(swigCPtr); } swigCPtr = 0; } } public void set_query(SWIGTYPE_p_float x) { swigfaissJNI.DistanceComputer_set_query(swigCPtr, this, SWIGTYPE_p_float.getCPtr(x)); } public float symmetric_dis(long i, long j) { return swigfaissJNI.DistanceComputer_symmetric_dis(swigCPtr, this, i, j); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/DoubleVector.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class DoubleVector { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected DoubleVector(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(DoubleVector obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_DoubleVector(swigCPtr); } swigCPtr = 0; } } public DoubleVector() { this(swigfaissJNI.new_DoubleVector(), true); } public void push_back(double arg0) { swigfaissJNI.DoubleVector_push_back(swigCPtr, this, arg0); } public void clear() { swigfaissJNI.DoubleVector_clear(swigCPtr, this); } public SWIGTYPE_p_double data() { long cPtr = swigfaissJNI.DoubleVector_data(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_double(cPtr, false); } public long size() { return swigfaissJNI.DoubleVector_size(swigCPtr, this); } public double at(long n) { return swigfaissJNI.DoubleVector_at(swigCPtr, this, n); } public void resize(long n) { swigfaissJNI.DoubleVector_resize(swigCPtr, this, n); } public void reserve(long n) { swigfaissJNI.DoubleVector_reserve(swigCPtr, this, n); } public void swap(DoubleVector other) { swigfaissJNI.DoubleVector_swap(swigCPtr, this, DoubleVector.getCPtr(other), other); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/FloatVector.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class FloatVector { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected FloatVector(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(FloatVector obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_FloatVector(swigCPtr); } swigCPtr = 0; } } public FloatVector() { this(swigfaissJNI.new_FloatVector(), true); } public void push_back(float arg0) { swigfaissJNI.FloatVector_push_back(swigCPtr, this, arg0); } public void clear() { swigfaissJNI.FloatVector_clear(swigCPtr, this); } public SWIGTYPE_p_float data() { long cPtr = swigfaissJNI.FloatVector_data(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } public long size() { return swigfaissJNI.FloatVector_size(swigCPtr, this); } public float at(long n) { return swigfaissJNI.FloatVector_at(swigCPtr, this, n); } public void resize(long n) { swigfaissJNI.FloatVector_resize(swigCPtr, this, n); } public void reserve(long n) { swigfaissJNI.FloatVector_reserve(swigCPtr, this, n); } public void swap(FloatVector other) { swigfaissJNI.FloatVector_swap(swigCPtr, this, FloatVector.getCPtr(other), other); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/FloatVectorVector.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class FloatVectorVector { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected FloatVectorVector(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(FloatVectorVector obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_FloatVectorVector(swigCPtr); } swigCPtr = 0; } } public FloatVectorVector() { this(swigfaissJNI.new_FloatVectorVector(), true); } public void push_back(FloatVector arg0) { swigfaissJNI.FloatVectorVector_push_back(swigCPtr, this, FloatVector.getCPtr(arg0), arg0); } public void clear() { swigfaissJNI.FloatVectorVector_clear(swigCPtr, this); } public FloatVector data() { long cPtr = swigfaissJNI.FloatVectorVector_data(swigCPtr, this); return (cPtr == 0) ? null : new FloatVector(cPtr, false); } public long size() { return swigfaissJNI.FloatVectorVector_size(swigCPtr, this); } public FloatVector at(long n) { return new FloatVector(swigfaissJNI.FloatVectorVector_at(swigCPtr, this, n), true); } public void resize(long n) { swigfaissJNI.FloatVectorVector_resize(swigCPtr, this, n); } public void reserve(long n) { swigfaissJNI.FloatVectorVector_reserve(swigCPtr, this, n); } public void swap(FloatVectorVector other) { swigfaissJNI.FloatVectorVector_swap(swigCPtr, this, FloatVectorVector.getCPtr(other), other); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/GenHammingComputer16.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class GenHammingComputer16 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected GenHammingComputer16(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(GenHammingComputer16 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_GenHammingComputer16(swigCPtr); } swigCPtr = 0; } } public void setA0(long value) { swigfaissJNI.GenHammingComputer16_a0_set(swigCPtr, this, value); } public long getA0() { return swigfaissJNI.GenHammingComputer16_a0_get(swigCPtr, this); } public void setA1(long value) { swigfaissJNI.GenHammingComputer16_a1_set(swigCPtr, this, value); } public long getA1() { return swigfaissJNI.GenHammingComputer16_a1_get(swigCPtr, this); } public GenHammingComputer16(SWIGTYPE_p_unsigned_char a8, int code_size) { this(swigfaissJNI.new_GenHammingComputer16(SWIGTYPE_p_unsigned_char.getCPtr(a8), code_size), true); } public int hamming(SWIGTYPE_p_unsigned_char b8) { return swigfaissJNI.GenHammingComputer16_hamming(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(b8)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/GenHammingComputer32.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class GenHammingComputer32 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected GenHammingComputer32(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(GenHammingComputer32 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_GenHammingComputer32(swigCPtr); } swigCPtr = 0; } } public void setA0(long value) { swigfaissJNI.GenHammingComputer32_a0_set(swigCPtr, this, value); } public long getA0() { return swigfaissJNI.GenHammingComputer32_a0_get(swigCPtr, this); } public void setA1(long value) { swigfaissJNI.GenHammingComputer32_a1_set(swigCPtr, this, value); } public long getA1() { return swigfaissJNI.GenHammingComputer32_a1_get(swigCPtr, this); } public void setA2(long value) { swigfaissJNI.GenHammingComputer32_a2_set(swigCPtr, this, value); } public long getA2() { return swigfaissJNI.GenHammingComputer32_a2_get(swigCPtr, this); } public void setA3(long value) { swigfaissJNI.GenHammingComputer32_a3_set(swigCPtr, this, value); } public long getA3() { return swigfaissJNI.GenHammingComputer32_a3_get(swigCPtr, this); } public GenHammingComputer32(SWIGTYPE_p_unsigned_char a8, int code_size) { this(swigfaissJNI.new_GenHammingComputer32(SWIGTYPE_p_unsigned_char.getCPtr(a8), code_size), true); } public int hamming(SWIGTYPE_p_unsigned_char b8) { return swigfaissJNI.GenHammingComputer32_hamming(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(b8)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/GenHammingComputer8.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class GenHammingComputer8 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected GenHammingComputer8(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(GenHammingComputer8 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_GenHammingComputer8(swigCPtr); } swigCPtr = 0; } } public void setA0(long value) { swigfaissJNI.GenHammingComputer8_a0_set(swigCPtr, this, value); } public long getA0() { return swigfaissJNI.GenHammingComputer8_a0_get(swigCPtr, this); } public GenHammingComputer8(SWIGTYPE_p_unsigned_char a, int code_size) { this(swigfaissJNI.new_GenHammingComputer8(SWIGTYPE_p_unsigned_char.getCPtr(a), code_size), true); } public int hamming(SWIGTYPE_p_unsigned_char b) { return swigfaissJNI.GenHammingComputer8_hamming(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(b)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/GenHammingComputerM8.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class GenHammingComputerM8 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected GenHammingComputerM8(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(GenHammingComputerM8 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_GenHammingComputerM8(swigCPtr); } swigCPtr = 0; } } public void setA(SWIGTYPE_p_unsigned_long value) { swigfaissJNI.GenHammingComputerM8_a_set(swigCPtr, this, SWIGTYPE_p_unsigned_long.getCPtr(value)); } public SWIGTYPE_p_unsigned_long getA() { long cPtr = swigfaissJNI.GenHammingComputerM8_a_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_long(cPtr, false); } public void setN(int value) { swigfaissJNI.GenHammingComputerM8_n_set(swigCPtr, this, value); } public int getN() { return swigfaissJNI.GenHammingComputerM8_n_get(swigCPtr, this); } public GenHammingComputerM8(SWIGTYPE_p_unsigned_char a8, int code_size) { this(swigfaissJNI.new_GenHammingComputerM8(SWIGTYPE_p_unsigned_char.getCPtr(a8), code_size), true); } public int hamming(SWIGTYPE_p_unsigned_char b8) { return swigfaissJNI.GenHammingComputerM8_hamming(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(b8)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/HNSW.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class HNSW { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected HNSW(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(HNSW obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_HNSW(swigCPtr); } swigCPtr = 0; } } static public class MinimaxHeap { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected MinimaxHeap(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(MinimaxHeap obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_HNSW_MinimaxHeap(swigCPtr); } swigCPtr = 0; } } public void setN(int value) { swigfaissJNI.HNSW_MinimaxHeap_n_set(swigCPtr, this, value); } public int getN() { return swigfaissJNI.HNSW_MinimaxHeap_n_get(swigCPtr, this); } public void setK(int value) { swigfaissJNI.HNSW_MinimaxHeap_k_set(swigCPtr, this, value); } public int getK() { return swigfaissJNI.HNSW_MinimaxHeap_k_get(swigCPtr, this); } public void setNvalid(int value) { swigfaissJNI.HNSW_MinimaxHeap_nvalid_set(swigCPtr, this, value); } public int getNvalid() { return swigfaissJNI.HNSW_MinimaxHeap_nvalid_get(swigCPtr, this); } public void setIds(IntVector value) { swigfaissJNI.HNSW_MinimaxHeap_ids_set(swigCPtr, this, IntVector.getCPtr(value), value); } public IntVector getIds() { long cPtr = swigfaissJNI.HNSW_MinimaxHeap_ids_get(swigCPtr, this); return (cPtr == 0) ? null : new IntVector(cPtr, false); } public void setDis(FloatVector value) { swigfaissJNI.HNSW_MinimaxHeap_dis_set(swigCPtr, this, FloatVector.getCPtr(value), value); } public FloatVector getDis() { long cPtr = swigfaissJNI.HNSW_MinimaxHeap_dis_get(swigCPtr, this); return (cPtr == 0) ? null : new FloatVector(cPtr, false); } public MinimaxHeap(int n) { this(swigfaissJNI.new_HNSW_MinimaxHeap(n), true); } public void push(int i, float v) { swigfaissJNI.HNSW_MinimaxHeap_push(swigCPtr, this, i, v); } public float max() { return swigfaissJNI.HNSW_MinimaxHeap_max(swigCPtr, this); } public int size() { return swigfaissJNI.HNSW_MinimaxHeap_size(swigCPtr, this); } public void clear() { swigfaissJNI.HNSW_MinimaxHeap_clear(swigCPtr, this); } public int pop_min(SWIGTYPE_p_float vmin_out) { return swigfaissJNI.HNSW_MinimaxHeap_pop_min__SWIG_0(swigCPtr, this, SWIGTYPE_p_float.getCPtr(vmin_out)); } public int pop_min() { return swigfaissJNI.HNSW_MinimaxHeap_pop_min__SWIG_1(swigCPtr, this); } public int count_below(float thresh) { return swigfaissJNI.HNSW_MinimaxHeap_count_below(swigCPtr, this, thresh); } } static public class NodeDistCloser { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected NodeDistCloser(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(NodeDistCloser obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_HNSW_NodeDistCloser(swigCPtr); } swigCPtr = 0; } } public void setD(float value) { swigfaissJNI.HNSW_NodeDistCloser_d_set(swigCPtr, this, value); } public float getD() { return swigfaissJNI.HNSW_NodeDistCloser_d_get(swigCPtr, this); } public void setId(int value) { swigfaissJNI.HNSW_NodeDistCloser_id_set(swigCPtr, this, value); } public int getId() { return swigfaissJNI.HNSW_NodeDistCloser_id_get(swigCPtr, this); } public NodeDistCloser(float d, int id) { this(swigfaissJNI.new_HNSW_NodeDistCloser(d, id), true); } } static public class NodeDistFarther { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected NodeDistFarther(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(NodeDistFarther obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_HNSW_NodeDistFarther(swigCPtr); } swigCPtr = 0; } } public void setD(float value) { swigfaissJNI.HNSW_NodeDistFarther_d_set(swigCPtr, this, value); } public float getD() { return swigfaissJNI.HNSW_NodeDistFarther_d_get(swigCPtr, this); } public void setId(int value) { swigfaissJNI.HNSW_NodeDistFarther_id_set(swigCPtr, this, value); } public int getId() { return swigfaissJNI.HNSW_NodeDistFarther_id_get(swigCPtr, this); } public NodeDistFarther(float d, int id) { this(swigfaissJNI.new_HNSW_NodeDistFarther(d, id), true); } } public void setAssign_probas(DoubleVector value) { swigfaissJNI.HNSW_assign_probas_set(swigCPtr, this, DoubleVector.getCPtr(value), value); } public DoubleVector getAssign_probas() { long cPtr = swigfaissJNI.HNSW_assign_probas_get(swigCPtr, this); return (cPtr == 0) ? null : new DoubleVector(cPtr, false); } public void setCum_nneighbor_per_level(IntVector value) { swigfaissJNI.HNSW_cum_nneighbor_per_level_set(swigCPtr, this, IntVector.getCPtr(value), value); } public IntVector getCum_nneighbor_per_level() { long cPtr = swigfaissJNI.HNSW_cum_nneighbor_per_level_get(swigCPtr, this); return (cPtr == 0) ? null : new IntVector(cPtr, false); } public void setLevels(IntVector value) { swigfaissJNI.HNSW_levels_set(swigCPtr, this, IntVector.getCPtr(value), value); } public IntVector getLevels() { long cPtr = swigfaissJNI.HNSW_levels_get(swigCPtr, this); return (cPtr == 0) ? null : new IntVector(cPtr, false); } public void setOffsets(Uint64Vector value) { swigfaissJNI.HNSW_offsets_set(swigCPtr, this, Uint64Vector.getCPtr(value), value); } public Uint64Vector getOffsets() { long cPtr = swigfaissJNI.HNSW_offsets_get(swigCPtr, this); return (cPtr == 0) ? null : new Uint64Vector(cPtr, false); } public void setNeighbors(IntVector value) { swigfaissJNI.HNSW_neighbors_set(swigCPtr, this, IntVector.getCPtr(value), value); } public IntVector getNeighbors() { long cPtr = swigfaissJNI.HNSW_neighbors_get(swigCPtr, this); return (cPtr == 0) ? null : new IntVector(cPtr, false); } public void setEntry_point(int value) { swigfaissJNI.HNSW_entry_point_set(swigCPtr, this, value); } public int getEntry_point() { return swigfaissJNI.HNSW_entry_point_get(swigCPtr, this); } public void setRng(SWIGTYPE_p_faiss__RandomGenerator value) { swigfaissJNI.HNSW_rng_set(swigCPtr, this, SWIGTYPE_p_faiss__RandomGenerator.getCPtr(value)); } public SWIGTYPE_p_faiss__RandomGenerator getRng() { long cPtr = swigfaissJNI.HNSW_rng_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_faiss__RandomGenerator(cPtr, false); } public void setMax_level(int value) { swigfaissJNI.HNSW_max_level_set(swigCPtr, this, value); } public int getMax_level() { return swigfaissJNI.HNSW_max_level_get(swigCPtr, this); } public void setEfConstruction(int value) { swigfaissJNI.HNSW_efConstruction_set(swigCPtr, this, value); } public int getEfConstruction() { return swigfaissJNI.HNSW_efConstruction_get(swigCPtr, this); } public void setEfSearch(int value) { swigfaissJNI.HNSW_efSearch_set(swigCPtr, this, value); } public int getEfSearch() { return swigfaissJNI.HNSW_efSearch_get(swigCPtr, this); } public void setCheck_relative_distance(boolean value) { swigfaissJNI.HNSW_check_relative_distance_set(swigCPtr, this, value); } public boolean getCheck_relative_distance() { return swigfaissJNI.HNSW_check_relative_distance_get(swigCPtr, this); } public void setUpper_beam(int value) { swigfaissJNI.HNSW_upper_beam_set(swigCPtr, this, value); } public int getUpper_beam() { return swigfaissJNI.HNSW_upper_beam_get(swigCPtr, this); } public void setSearch_bounded_queue(boolean value) { swigfaissJNI.HNSW_search_bounded_queue_set(swigCPtr, this, value); } public boolean getSearch_bounded_queue() { return swigfaissJNI.HNSW_search_bounded_queue_get(swigCPtr, this); } public void set_default_probas(int M, float levelMult) { swigfaissJNI.HNSW_set_default_probas(swigCPtr, this, M, levelMult); } public void set_nb_neighbors(int level_no, int n) { swigfaissJNI.HNSW_set_nb_neighbors(swigCPtr, this, level_no, n); } public int nb_neighbors(int layer_no) { return swigfaissJNI.HNSW_nb_neighbors(swigCPtr, this, layer_no); } public int cum_nb_neighbors(int layer_no) { return swigfaissJNI.HNSW_cum_nb_neighbors(swigCPtr, this, layer_no); } public void neighbor_range(long no, int layer_no, SWIGTYPE_p_unsigned_long begin, SWIGTYPE_p_unsigned_long end) { swigfaissJNI.HNSW_neighbor_range(swigCPtr, this, no, layer_no, SWIGTYPE_p_unsigned_long.getCPtr(begin), SWIGTYPE_p_unsigned_long.getCPtr(end)); } public HNSW(int M) { this(swigfaissJNI.new_HNSW__SWIG_0(M), true); } public HNSW() { this(swigfaissJNI.new_HNSW__SWIG_1(), true); } public int random_level() { return swigfaissJNI.HNSW_random_level(swigCPtr, this); } public void fill_with_random_links(long n) { swigfaissJNI.HNSW_fill_with_random_links(swigCPtr, this, n); } public void add_links_starting_from(DistanceComputer ptdis, int pt_id, int nearest, float d_nearest, int level, SWIGTYPE_p_omp_lock_t locks, VisitedTable vt) { swigfaissJNI.HNSW_add_links_starting_from(swigCPtr, this, DistanceComputer.getCPtr(ptdis), ptdis, pt_id, nearest, d_nearest, level, SWIGTYPE_p_omp_lock_t.getCPtr(locks), VisitedTable.getCPtr(vt), vt); } public void add_with_locks(DistanceComputer ptdis, int pt_level, int pt_id, SWIGTYPE_p_std__vectorT_omp_lock_t_t locks, VisitedTable vt) { swigfaissJNI.HNSW_add_with_locks(swigCPtr, this, DistanceComputer.getCPtr(ptdis), ptdis, pt_level, pt_id, SWIGTYPE_p_std__vectorT_omp_lock_t_t.getCPtr(locks), VisitedTable.getCPtr(vt), vt); } public int search_from_candidates(DistanceComputer qdis, int k, LongVector I, SWIGTYPE_p_float D, HNSW.MinimaxHeap candidates, VisitedTable vt, HNSWStats stats, int level, int nres_in) { return swigfaissJNI.HNSW_search_from_candidates__SWIG_0(swigCPtr, this, DistanceComputer.getCPtr(qdis), qdis, k, SWIGTYPE_p_long_long.getCPtr(I.data()), I, SWIGTYPE_p_float.getCPtr(D), HNSW.MinimaxHeap.getCPtr(candidates), candidates, VisitedTable.getCPtr(vt), vt, HNSWStats.getCPtr(stats), stats, level, nres_in); } public int search_from_candidates(DistanceComputer qdis, int k, LongVector I, SWIGTYPE_p_float D, HNSW.MinimaxHeap candidates, VisitedTable vt, HNSWStats stats, int level) { return swigfaissJNI.HNSW_search_from_candidates__SWIG_1(swigCPtr, this, DistanceComputer.getCPtr(qdis), qdis, k, SWIGTYPE_p_long_long.getCPtr(I.data()), I, SWIGTYPE_p_float.getCPtr(D), HNSW.MinimaxHeap.getCPtr(candidates), candidates, VisitedTable.getCPtr(vt), vt, HNSWStats.getCPtr(stats), stats, level); } public SWIGTYPE_p_std__priority_queueT_std__pairT_float_int_t_t search_from_candidate_unbounded(SWIGTYPE_p_std__pairT_float_int_t node, DistanceComputer qdis, int ef, VisitedTable vt, HNSWStats stats) { return new SWIGTYPE_p_std__priority_queueT_std__pairT_float_int_t_t(swigfaissJNI.HNSW_search_from_candidate_unbounded(swigCPtr, this, SWIGTYPE_p_std__pairT_float_int_t.getCPtr(node), DistanceComputer.getCPtr(qdis), qdis, ef, VisitedTable.getCPtr(vt), vt, HNSWStats.getCPtr(stats), stats), true); } public HNSWStats search(DistanceComputer qdis, int k, LongVector I, SWIGTYPE_p_float D, VisitedTable vt) { return new HNSWStats(swigfaissJNI.HNSW_search(swigCPtr, this, DistanceComputer.getCPtr(qdis), qdis, k, SWIGTYPE_p_long_long.getCPtr(I.data()), I, SWIGTYPE_p_float.getCPtr(D), VisitedTable.getCPtr(vt), vt), true); } public void reset() { swigfaissJNI.HNSW_reset(swigCPtr, this); } public void clear_neighbor_tables(int level) { swigfaissJNI.HNSW_clear_neighbor_tables(swigCPtr, this, level); } public void print_neighbor_stats(int level) { swigfaissJNI.HNSW_print_neighbor_stats(swigCPtr, this, level); } public int prepare_level_tab(long n, boolean preset_levels) { return swigfaissJNI.HNSW_prepare_level_tab__SWIG_0(swigCPtr, this, n, preset_levels); } public int prepare_level_tab(long n) { return swigfaissJNI.HNSW_prepare_level_tab__SWIG_1(swigCPtr, this, n); } public static void shrink_neighbor_list(DistanceComputer qdis, SWIGTYPE_p_std__priority_queueT_faiss__HNSW__NodeDistFarther_t input, SWIGTYPE_p_std__vectorT_faiss__HNSW__NodeDistFarther_t output, int max_size) { swigfaissJNI.HNSW_shrink_neighbor_list(DistanceComputer.getCPtr(qdis), qdis, SWIGTYPE_p_std__priority_queueT_faiss__HNSW__NodeDistFarther_t.getCPtr(input), SWIGTYPE_p_std__vectorT_faiss__HNSW__NodeDistFarther_t.getCPtr(output), max_size); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/HNSWStats.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class HNSWStats { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected HNSWStats(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(HNSWStats obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_HNSWStats(swigCPtr); } swigCPtr = 0; } } public void setN1(long value) { swigfaissJNI.HNSWStats_n1_set(swigCPtr, this, value); } public long getN1() { return swigfaissJNI.HNSWStats_n1_get(swigCPtr, this); } public void setN2(long value) { swigfaissJNI.HNSWStats_n2_set(swigCPtr, this, value); } public long getN2() { return swigfaissJNI.HNSWStats_n2_get(swigCPtr, this); } public void setN3(long value) { swigfaissJNI.HNSWStats_n3_set(swigCPtr, this, value); } public long getN3() { return swigfaissJNI.HNSWStats_n3_get(swigCPtr, this); } public void setNdis(long value) { swigfaissJNI.HNSWStats_ndis_set(swigCPtr, this, value); } public long getNdis() { return swigfaissJNI.HNSWStats_ndis_get(swigCPtr, this); } public void setNreorder(long value) { swigfaissJNI.HNSWStats_nreorder_set(swigCPtr, this, value); } public long getNreorder() { return swigfaissJNI.HNSWStats_nreorder_get(swigCPtr, this); } public HNSWStats(long n1, long n2, long n3, long ndis, long nreorder) { this(swigfaissJNI.new_HNSWStats__SWIG_0(n1, n2, n3, ndis, nreorder), true); } public HNSWStats(long n1, long n2, long n3, long ndis) { this(swigfaissJNI.new_HNSWStats__SWIG_1(n1, n2, n3, ndis), true); } public HNSWStats(long n1, long n2, long n3) { this(swigfaissJNI.new_HNSWStats__SWIG_2(n1, n2, n3), true); } public HNSWStats(long n1, long n2) { this(swigfaissJNI.new_HNSWStats__SWIG_3(n1, n2), true); } public HNSWStats(long n1) { this(swigfaissJNI.new_HNSWStats__SWIG_4(n1), true); } public HNSWStats() { this(swigfaissJNI.new_HNSWStats__SWIG_5(), true); } public void reset() { swigfaissJNI.HNSWStats_reset(swigCPtr, this); } public void combine(HNSWStats other) { swigfaissJNI.HNSWStats_combine(swigCPtr, this, HNSWStats.getCPtr(other), other); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/HStackInvertedLists.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class HStackInvertedLists extends ReadOnlyInvertedLists { private transient long swigCPtr; protected HStackInvertedLists(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.HStackInvertedLists_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(HStackInvertedLists obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_HStackInvertedLists(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setIls(SWIGTYPE_p_std__vectorT_faiss__InvertedLists_const_p_t value) { swigfaissJNI.HStackInvertedLists_ils_set(swigCPtr, this, SWIGTYPE_p_std__vectorT_faiss__InvertedLists_const_p_t.getCPtr(value)); } public SWIGTYPE_p_std__vectorT_faiss__InvertedLists_const_p_t getIls() { long cPtr = swigfaissJNI.HStackInvertedLists_ils_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_faiss__InvertedLists_const_p_t(cPtr, false); } public HStackInvertedLists(int nil, SWIGTYPE_p_p_faiss__InvertedLists ils) { this(swigfaissJNI.new_HStackInvertedLists(nil, SWIGTYPE_p_p_faiss__InvertedLists.getCPtr(ils)), true); } public long list_size(long list_no) { return swigfaissJNI.HStackInvertedLists_list_size(swigCPtr, this, list_no); } public SWIGTYPE_p_unsigned_char get_codes(long list_no) { long cPtr = swigfaissJNI.HStackInvertedLists_get_codes(swigCPtr, this, list_no); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public LongVector get_ids(long list_no) { return new LongVector(swigfaissJNI.HStackInvertedLists_get_ids(swigCPtr, this, list_no), false); } public void prefetch_lists(LongVector list_nos, int nlist) { swigfaissJNI.HStackInvertedLists_prefetch_lists(swigCPtr, this, SWIGTYPE_p_long_long.getCPtr(list_nos.data()), list_nos, nlist); } public void release_codes(long list_no, SWIGTYPE_p_unsigned_char codes) { swigfaissJNI.HStackInvertedLists_release_codes(swigCPtr, this, list_no, SWIGTYPE_p_unsigned_char.getCPtr(codes)); } public void release_ids(long list_no, LongVector ids) { swigfaissJNI.HStackInvertedLists_release_ids(swigCPtr, this, list_no, SWIGTYPE_p_long_long.getCPtr(ids.data()), ids); } public long get_single_id(long list_no, long offset) { return swigfaissJNI.HStackInvertedLists_get_single_id(swigCPtr, this, list_no, offset); } public SWIGTYPE_p_unsigned_char get_single_code(long list_no, long offset) { long cPtr = swigfaissJNI.HStackInvertedLists_get_single_code(swigCPtr, this, list_no, offset); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/HammingComputer16.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class HammingComputer16 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected HammingComputer16(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(HammingComputer16 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_HammingComputer16(swigCPtr); } swigCPtr = 0; } } public void setA0(long value) { swigfaissJNI.HammingComputer16_a0_set(swigCPtr, this, value); } public long getA0() { return swigfaissJNI.HammingComputer16_a0_get(swigCPtr, this); } public void setA1(long value) { swigfaissJNI.HammingComputer16_a1_set(swigCPtr, this, value); } public long getA1() { return swigfaissJNI.HammingComputer16_a1_get(swigCPtr, this); } public HammingComputer16() { this(swigfaissJNI.new_HammingComputer16__SWIG_0(), true); } public HammingComputer16(SWIGTYPE_p_unsigned_char a8, int code_size) { this(swigfaissJNI.new_HammingComputer16__SWIG_1(SWIGTYPE_p_unsigned_char.getCPtr(a8), code_size), true); } public void set(SWIGTYPE_p_unsigned_char a8, int code_size) { swigfaissJNI.HammingComputer16_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(a8), code_size); } public int hamming(SWIGTYPE_p_unsigned_char b8) { return swigfaissJNI.HammingComputer16_hamming(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(b8)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/HammingComputer20.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class HammingComputer20 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected HammingComputer20(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(HammingComputer20 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_HammingComputer20(swigCPtr); } swigCPtr = 0; } } public void setA0(long value) { swigfaissJNI.HammingComputer20_a0_set(swigCPtr, this, value); } public long getA0() { return swigfaissJNI.HammingComputer20_a0_get(swigCPtr, this); } public void setA1(long value) { swigfaissJNI.HammingComputer20_a1_set(swigCPtr, this, value); } public long getA1() { return swigfaissJNI.HammingComputer20_a1_get(swigCPtr, this); } public void setA2(SWIGTYPE_p_uint32_t value) { swigfaissJNI.HammingComputer20_a2_set(swigCPtr, this, SWIGTYPE_p_uint32_t.getCPtr(value)); } public SWIGTYPE_p_uint32_t getA2() { return new SWIGTYPE_p_uint32_t(swigfaissJNI.HammingComputer20_a2_get(swigCPtr, this), true); } public HammingComputer20() { this(swigfaissJNI.new_HammingComputer20__SWIG_0(), true); } public HammingComputer20(SWIGTYPE_p_unsigned_char a8, int code_size) { this(swigfaissJNI.new_HammingComputer20__SWIG_1(SWIGTYPE_p_unsigned_char.getCPtr(a8), code_size), true); } public void set(SWIGTYPE_p_unsigned_char a8, int code_size) { swigfaissJNI.HammingComputer20_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(a8), code_size); } public int hamming(SWIGTYPE_p_unsigned_char b8) { return swigfaissJNI.HammingComputer20_hamming(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(b8)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/HammingComputer32.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class HammingComputer32 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected HammingComputer32(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(HammingComputer32 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_HammingComputer32(swigCPtr); } swigCPtr = 0; } } public void setA0(long value) { swigfaissJNI.HammingComputer32_a0_set(swigCPtr, this, value); } public long getA0() { return swigfaissJNI.HammingComputer32_a0_get(swigCPtr, this); } public void setA1(long value) { swigfaissJNI.HammingComputer32_a1_set(swigCPtr, this, value); } public long getA1() { return swigfaissJNI.HammingComputer32_a1_get(swigCPtr, this); } public void setA2(long value) { swigfaissJNI.HammingComputer32_a2_set(swigCPtr, this, value); } public long getA2() { return swigfaissJNI.HammingComputer32_a2_get(swigCPtr, this); } public void setA3(long value) { swigfaissJNI.HammingComputer32_a3_set(swigCPtr, this, value); } public long getA3() { return swigfaissJNI.HammingComputer32_a3_get(swigCPtr, this); } public HammingComputer32() { this(swigfaissJNI.new_HammingComputer32__SWIG_0(), true); } public HammingComputer32(SWIGTYPE_p_unsigned_char a8, int code_size) { this(swigfaissJNI.new_HammingComputer32__SWIG_1(SWIGTYPE_p_unsigned_char.getCPtr(a8), code_size), true); } public void set(SWIGTYPE_p_unsigned_char a8, int code_size) { swigfaissJNI.HammingComputer32_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(a8), code_size); } public int hamming(SWIGTYPE_p_unsigned_char b8) { return swigfaissJNI.HammingComputer32_hamming(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(b8)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/HammingComputer4.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class HammingComputer4 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected HammingComputer4(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(HammingComputer4 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_HammingComputer4(swigCPtr); } swigCPtr = 0; } } public void setA0(SWIGTYPE_p_uint32_t value) { swigfaissJNI.HammingComputer4_a0_set(swigCPtr, this, SWIGTYPE_p_uint32_t.getCPtr(value)); } public SWIGTYPE_p_uint32_t getA0() { return new SWIGTYPE_p_uint32_t(swigfaissJNI.HammingComputer4_a0_get(swigCPtr, this), true); } public HammingComputer4() { this(swigfaissJNI.new_HammingComputer4__SWIG_0(), true); } public HammingComputer4(SWIGTYPE_p_unsigned_char a, int code_size) { this(swigfaissJNI.new_HammingComputer4__SWIG_1(SWIGTYPE_p_unsigned_char.getCPtr(a), code_size), true); } public void set(SWIGTYPE_p_unsigned_char a, int code_size) { swigfaissJNI.HammingComputer4_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(a), code_size); } public int hamming(SWIGTYPE_p_unsigned_char b) { return swigfaissJNI.HammingComputer4_hamming(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(b)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/HammingComputer64.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class HammingComputer64 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected HammingComputer64(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(HammingComputer64 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_HammingComputer64(swigCPtr); } swigCPtr = 0; } } public void setA0(long value) { swigfaissJNI.HammingComputer64_a0_set(swigCPtr, this, value); } public long getA0() { return swigfaissJNI.HammingComputer64_a0_get(swigCPtr, this); } public void setA1(long value) { swigfaissJNI.HammingComputer64_a1_set(swigCPtr, this, value); } public long getA1() { return swigfaissJNI.HammingComputer64_a1_get(swigCPtr, this); } public void setA2(long value) { swigfaissJNI.HammingComputer64_a2_set(swigCPtr, this, value); } public long getA2() { return swigfaissJNI.HammingComputer64_a2_get(swigCPtr, this); } public void setA3(long value) { swigfaissJNI.HammingComputer64_a3_set(swigCPtr, this, value); } public long getA3() { return swigfaissJNI.HammingComputer64_a3_get(swigCPtr, this); } public void setA4(long value) { swigfaissJNI.HammingComputer64_a4_set(swigCPtr, this, value); } public long getA4() { return swigfaissJNI.HammingComputer64_a4_get(swigCPtr, this); } public void setA5(long value) { swigfaissJNI.HammingComputer64_a5_set(swigCPtr, this, value); } public long getA5() { return swigfaissJNI.HammingComputer64_a5_get(swigCPtr, this); } public void setA6(long value) { swigfaissJNI.HammingComputer64_a6_set(swigCPtr, this, value); } public long getA6() { return swigfaissJNI.HammingComputer64_a6_get(swigCPtr, this); } public void setA7(long value) { swigfaissJNI.HammingComputer64_a7_set(swigCPtr, this, value); } public long getA7() { return swigfaissJNI.HammingComputer64_a7_get(swigCPtr, this); } public HammingComputer64() { this(swigfaissJNI.new_HammingComputer64__SWIG_0(), true); } public HammingComputer64(SWIGTYPE_p_unsigned_char a8, int code_size) { this(swigfaissJNI.new_HammingComputer64__SWIG_1(SWIGTYPE_p_unsigned_char.getCPtr(a8), code_size), true); } public void set(SWIGTYPE_p_unsigned_char a8, int code_size) { swigfaissJNI.HammingComputer64_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(a8), code_size); } public int hamming(SWIGTYPE_p_unsigned_char b8) { return swigfaissJNI.HammingComputer64_hamming(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(b8)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/HammingComputer8.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class HammingComputer8 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected HammingComputer8(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(HammingComputer8 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_HammingComputer8(swigCPtr); } swigCPtr = 0; } } public void setA0(long value) { swigfaissJNI.HammingComputer8_a0_set(swigCPtr, this, value); } public long getA0() { return swigfaissJNI.HammingComputer8_a0_get(swigCPtr, this); } public HammingComputer8() { this(swigfaissJNI.new_HammingComputer8__SWIG_0(), true); } public HammingComputer8(SWIGTYPE_p_unsigned_char a, int code_size) { this(swigfaissJNI.new_HammingComputer8__SWIG_1(SWIGTYPE_p_unsigned_char.getCPtr(a), code_size), true); } public void set(SWIGTYPE_p_unsigned_char a, int code_size) { swigfaissJNI.HammingComputer8_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(a), code_size); } public int hamming(SWIGTYPE_p_unsigned_char b) { return swigfaissJNI.HammingComputer8_hamming(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(b)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/HammingComputerDefault.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class HammingComputerDefault { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected HammingComputerDefault(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(HammingComputerDefault obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_HammingComputerDefault(swigCPtr); } swigCPtr = 0; } } public void setA8(SWIGTYPE_p_unsigned_char value) { swigfaissJNI.HammingComputerDefault_a8_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(value)); } public SWIGTYPE_p_unsigned_char getA8() { long cPtr = swigfaissJNI.HammingComputerDefault_a8_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public void setQuotient8(int value) { swigfaissJNI.HammingComputerDefault_quotient8_set(swigCPtr, this, value); } public int getQuotient8() { return swigfaissJNI.HammingComputerDefault_quotient8_get(swigCPtr, this); } public void setRemainder8(int value) { swigfaissJNI.HammingComputerDefault_remainder8_set(swigCPtr, this, value); } public int getRemainder8() { return swigfaissJNI.HammingComputerDefault_remainder8_get(swigCPtr, this); } public HammingComputerDefault() { this(swigfaissJNI.new_HammingComputerDefault__SWIG_0(), true); } public HammingComputerDefault(SWIGTYPE_p_unsigned_char a8, int code_size) { this(swigfaissJNI.new_HammingComputerDefault__SWIG_1(SWIGTYPE_p_unsigned_char.getCPtr(a8), code_size), true); } public void set(SWIGTYPE_p_unsigned_char a8, int code_size) { swigfaissJNI.HammingComputerDefault_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(a8), code_size); } public int hamming(SWIGTYPE_p_unsigned_char b8) { return swigfaissJNI.HammingComputerDefault_hamming(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(b8)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/HammingComputerM4.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class HammingComputerM4 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected HammingComputerM4(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(HammingComputerM4 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_HammingComputerM4(swigCPtr); } swigCPtr = 0; } } public void setA(SWIGTYPE_p_uint32_t value) { swigfaissJNI.HammingComputerM4_a_set(swigCPtr, this, SWIGTYPE_p_uint32_t.getCPtr(value)); } public SWIGTYPE_p_uint32_t getA() { long cPtr = swigfaissJNI.HammingComputerM4_a_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_uint32_t(cPtr, false); } public void setN(int value) { swigfaissJNI.HammingComputerM4_n_set(swigCPtr, this, value); } public int getN() { return swigfaissJNI.HammingComputerM4_n_get(swigCPtr, this); } public HammingComputerM4() { this(swigfaissJNI.new_HammingComputerM4__SWIG_0(), true); } public HammingComputerM4(SWIGTYPE_p_unsigned_char a4, int code_size) { this(swigfaissJNI.new_HammingComputerM4__SWIG_1(SWIGTYPE_p_unsigned_char.getCPtr(a4), code_size), true); } public void set(SWIGTYPE_p_unsigned_char a4, int code_size) { swigfaissJNI.HammingComputerM4_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(a4), code_size); } public int hamming(SWIGTYPE_p_unsigned_char b8) { return swigfaissJNI.HammingComputerM4_hamming(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(b8)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/HammingComputerM8.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class HammingComputerM8 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected HammingComputerM8(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(HammingComputerM8 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_HammingComputerM8(swigCPtr); } swigCPtr = 0; } } public void setA(SWIGTYPE_p_unsigned_long value) { swigfaissJNI.HammingComputerM8_a_set(swigCPtr, this, SWIGTYPE_p_unsigned_long.getCPtr(value)); } public SWIGTYPE_p_unsigned_long getA() { long cPtr = swigfaissJNI.HammingComputerM8_a_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_long(cPtr, false); } public void setN(int value) { swigfaissJNI.HammingComputerM8_n_set(swigCPtr, this, value); } public int getN() { return swigfaissJNI.HammingComputerM8_n_get(swigCPtr, this); } public HammingComputerM8() { this(swigfaissJNI.new_HammingComputerM8__SWIG_0(), true); } public HammingComputerM8(SWIGTYPE_p_unsigned_char a8, int code_size) { this(swigfaissJNI.new_HammingComputerM8__SWIG_1(SWIGTYPE_p_unsigned_char.getCPtr(a8), code_size), true); } public void set(SWIGTYPE_p_unsigned_char a8, int code_size) { swigfaissJNI.HammingComputerM8_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(a8), code_size); } public int hamming(SWIGTYPE_p_unsigned_char b8) { return swigfaissJNI.HammingComputerM8_hamming(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(b8)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IDSelector.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IDSelector { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected IDSelector(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(IDSelector obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IDSelector(swigCPtr); } swigCPtr = 0; } } public boolean is_member(long id) { return swigfaissJNI.IDSelector_is_member(swigCPtr, this, id); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IDSelectorArray.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IDSelectorArray extends IDSelector { private transient long swigCPtr; protected IDSelectorArray(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IDSelectorArray_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IDSelectorArray obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IDSelectorArray(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setN(long value) { swigfaissJNI.IDSelectorArray_n_set(swigCPtr, this, value); } public long getN() { return swigfaissJNI.IDSelectorArray_n_get(swigCPtr, this); } public void setIds(LongVector value) { swigfaissJNI.IDSelectorArray_ids_set(swigCPtr, this, SWIGTYPE_p_long_long.getCPtr(value.data()), value); } public LongVector getIds() { return new LongVector(swigfaissJNI.IDSelectorArray_ids_get(swigCPtr, this), false); } public IDSelectorArray(long n, LongVector ids) { this(swigfaissJNI.new_IDSelectorArray(n, SWIGTYPE_p_long_long.getCPtr(ids.data()), ids), true); } public boolean is_member(long id) { return swigfaissJNI.IDSelectorArray_is_member(swigCPtr, this, id); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IDSelectorBatch.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IDSelectorBatch extends IDSelector { private transient long swigCPtr; protected IDSelectorBatch(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IDSelectorBatch_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IDSelectorBatch obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IDSelectorBatch(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setNbits(int value) { swigfaissJNI.IDSelectorBatch_nbits_set(swigCPtr, this, value); } public int getNbits() { return swigfaissJNI.IDSelectorBatch_nbits_get(swigCPtr, this); } public void setMask(long value) { swigfaissJNI.IDSelectorBatch_mask_set(swigCPtr, this, value); } public long getMask() { return swigfaissJNI.IDSelectorBatch_mask_get(swigCPtr, this); } public IDSelectorBatch(long n, LongVector indices) { this(swigfaissJNI.new_IDSelectorBatch(n, SWIGTYPE_p_long_long.getCPtr(indices.data()), indices), true); } public boolean is_member(long id) { return swigfaissJNI.IDSelectorBatch_is_member(swigCPtr, this, id); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IDSelectorRange.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IDSelectorRange extends IDSelector { private transient long swigCPtr; protected IDSelectorRange(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IDSelectorRange_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IDSelectorRange obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IDSelectorRange(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setImin(long value) { swigfaissJNI.IDSelectorRange_imin_set(swigCPtr, this, value); } public long getImin() { return swigfaissJNI.IDSelectorRange_imin_get(swigCPtr, this); } public void setImax(long value) { swigfaissJNI.IDSelectorRange_imax_set(swigCPtr, this, value); } public long getImax() { return swigfaissJNI.IDSelectorRange_imax_get(swigCPtr, this); } public IDSelectorRange(long imin, long imax) { this(swigfaissJNI.new_IDSelectorRange(imin, imax), true); } public boolean is_member(long id) { return swigfaissJNI.IDSelectorRange_is_member(swigCPtr, this, id); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/ITQMatrix.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class ITQMatrix extends LinearTransform { private transient long swigCPtr; protected ITQMatrix(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.ITQMatrix_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(ITQMatrix obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_ITQMatrix(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setMax_iter(int value) { swigfaissJNI.ITQMatrix_max_iter_set(swigCPtr, this, value); } public int getMax_iter() { return swigfaissJNI.ITQMatrix_max_iter_get(swigCPtr, this); } public void setSeed(int value) { swigfaissJNI.ITQMatrix_seed_set(swigCPtr, this, value); } public int getSeed() { return swigfaissJNI.ITQMatrix_seed_get(swigCPtr, this); } public void setInit_rotation(DoubleVector value) { swigfaissJNI.ITQMatrix_init_rotation_set(swigCPtr, this, DoubleVector.getCPtr(value), value); } public DoubleVector getInit_rotation() { long cPtr = swigfaissJNI.ITQMatrix_init_rotation_get(swigCPtr, this); return (cPtr == 0) ? null : new DoubleVector(cPtr, false); } public ITQMatrix(int d) { this(swigfaissJNI.new_ITQMatrix__SWIG_0(d), true); } public ITQMatrix() { this(swigfaissJNI.new_ITQMatrix__SWIG_1(), true); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.ITQMatrix_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/ITQTransform.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class ITQTransform extends VectorTransform { private transient long swigCPtr; protected ITQTransform(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.ITQTransform_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(ITQTransform obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_ITQTransform(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setMean(FloatVector value) { swigfaissJNI.ITQTransform_mean_set(swigCPtr, this, FloatVector.getCPtr(value), value); } public FloatVector getMean() { long cPtr = swigfaissJNI.ITQTransform_mean_get(swigCPtr, this); return (cPtr == 0) ? null : new FloatVector(cPtr, false); } public void setDo_pca(boolean value) { swigfaissJNI.ITQTransform_do_pca_set(swigCPtr, this, value); } public boolean getDo_pca() { return swigfaissJNI.ITQTransform_do_pca_get(swigCPtr, this); } public void setItq(ITQMatrix value) { swigfaissJNI.ITQTransform_itq_set(swigCPtr, this, ITQMatrix.getCPtr(value), value); } public ITQMatrix getItq() { long cPtr = swigfaissJNI.ITQTransform_itq_get(swigCPtr, this); return (cPtr == 0) ? null : new ITQMatrix(cPtr, false); } public void setMax_train_per_dim(int value) { swigfaissJNI.ITQTransform_max_train_per_dim_set(swigCPtr, this, value); } public int getMax_train_per_dim() { return swigfaissJNI.ITQTransform_max_train_per_dim_get(swigCPtr, this); } public void setPca_then_itq(LinearTransform value) { swigfaissJNI.ITQTransform_pca_then_itq_set(swigCPtr, this, LinearTransform.getCPtr(value), value); } public LinearTransform getPca_then_itq() { long cPtr = swigfaissJNI.ITQTransform_pca_then_itq_get(swigCPtr, this); return (cPtr == 0) ? null : new LinearTransform(cPtr, false); } public ITQTransform(int d_in, int d_out, boolean do_pca) { this(swigfaissJNI.new_ITQTransform__SWIG_0(d_in, d_out, do_pca), true); } public ITQTransform(int d_in, int d_out) { this(swigfaissJNI.new_ITQTransform__SWIG_1(d_in, d_out), true); } public ITQTransform(int d_in) { this(swigfaissJNI.new_ITQTransform__SWIG_2(d_in), true); } public ITQTransform() { this(swigfaissJNI.new_ITQTransform__SWIG_3(), true); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.ITQTransform_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void apply_noalloc(long n, SWIGTYPE_p_float x, SWIGTYPE_p_float xt) { swigfaissJNI.ITQTransform_apply_noalloc(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_float.getCPtr(xt)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IVFPQSearchParameters.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IVFPQSearchParameters extends IVFSearchParameters { private transient long swigCPtr; protected IVFPQSearchParameters(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IVFPQSearchParameters_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IVFPQSearchParameters obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IVFPQSearchParameters(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setScan_table_threshold(long value) { swigfaissJNI.IVFPQSearchParameters_scan_table_threshold_set(swigCPtr, this, value); } public long getScan_table_threshold() { return swigfaissJNI.IVFPQSearchParameters_scan_table_threshold_get(swigCPtr, this); } public void setPolysemous_ht(int value) { swigfaissJNI.IVFPQSearchParameters_polysemous_ht_set(swigCPtr, this, value); } public int getPolysemous_ht() { return swigfaissJNI.IVFPQSearchParameters_polysemous_ht_get(swigCPtr, this); } public IVFPQSearchParameters() { this(swigfaissJNI.new_IVFPQSearchParameters(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IVFSearchParameters.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IVFSearchParameters { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected IVFSearchParameters(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(IVFSearchParameters obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IVFSearchParameters(swigCPtr); } swigCPtr = 0; } } public void setNprobe(long value) { swigfaissJNI.IVFSearchParameters_nprobe_set(swigCPtr, this, value); } public long getNprobe() { return swigfaissJNI.IVFSearchParameters_nprobe_get(swigCPtr, this); } public void setMax_codes(long value) { swigfaissJNI.IVFSearchParameters_max_codes_set(swigCPtr, this, value); } public long getMax_codes() { return swigfaissJNI.IVFSearchParameters_max_codes_get(swigCPtr, this); } public IVFSearchParameters() { this(swigfaissJNI.new_IVFSearchParameters(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/Index.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class Index { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected Index(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(Index obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_Index(swigCPtr); } swigCPtr = 0; } } public void setD(int value) { swigfaissJNI.Index_d_set(swigCPtr, this, value); } public int getD() { return swigfaissJNI.Index_d_get(swigCPtr, this); } public void setNtotal(long value) { swigfaissJNI.Index_ntotal_set(swigCPtr, this, value); } public long getNtotal() { return swigfaissJNI.Index_ntotal_get(swigCPtr, this); } public void setVerbose(boolean value) { swigfaissJNI.Index_verbose_set(swigCPtr, this, value); } public boolean getVerbose() { return swigfaissJNI.Index_verbose_get(swigCPtr, this); } public void setIs_trained(boolean value) { swigfaissJNI.Index_is_trained_set(swigCPtr, this, value); } public boolean getIs_trained() { return swigfaissJNI.Index_is_trained_get(swigCPtr, this); } public void setMetric_type(MetricType value) { swigfaissJNI.Index_metric_type_set(swigCPtr, this, value.swigValue()); } public MetricType getMetric_type() { return MetricType.swigToEnum(swigfaissJNI.Index_metric_type_get(swigCPtr, this)); } public void setMetric_arg(float value) { swigfaissJNI.Index_metric_arg_set(swigCPtr, this, value); } public float getMetric_arg() { return swigfaissJNI.Index_metric_arg_get(swigCPtr, this); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.Index_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void add(long n, SWIGTYPE_p_float x) { swigfaissJNI.Index_add(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void add_with_ids(long n, SWIGTYPE_p_float x, LongVector xids) { swigfaissJNI.Index_add_with_ids(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(xids.data()), xids); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.Index_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void range_search(long n, SWIGTYPE_p_float x, float radius, RangeSearchResult result) { swigfaissJNI.Index_range_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), radius, RangeSearchResult.getCPtr(result), result); } public void assign(long n, SWIGTYPE_p_float x, LongVector labels, long k) { swigfaissJNI.Index_assign__SWIG_0(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, k); } public void assign(long n, SWIGTYPE_p_float x, LongVector labels) { swigfaissJNI.Index_assign__SWIG_1(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void reset() { swigfaissJNI.Index_reset(swigCPtr, this); } public long remove_ids(IDSelector sel) { return swigfaissJNI.Index_remove_ids(swigCPtr, this, IDSelector.getCPtr(sel), sel); } public void reconstruct(long key, SWIGTYPE_p_float recons) { swigfaissJNI.Index_reconstruct(swigCPtr, this, key, SWIGTYPE_p_float.getCPtr(recons)); } public void reconstruct_n(long i0, long ni, SWIGTYPE_p_float recons) { swigfaissJNI.Index_reconstruct_n(swigCPtr, this, i0, ni, SWIGTYPE_p_float.getCPtr(recons)); } public void search_and_reconstruct(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels, SWIGTYPE_p_float recons) { swigfaissJNI.Index_search_and_reconstruct(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, SWIGTYPE_p_float.getCPtr(recons)); } public void compute_residual(SWIGTYPE_p_float x, SWIGTYPE_p_float residual, long key) { swigfaissJNI.Index_compute_residual(swigCPtr, this, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_float.getCPtr(residual), key); } public void compute_residual_n(long n, SWIGTYPE_p_float xs, SWIGTYPE_p_float residuals, LongVector keys) { swigfaissJNI.Index_compute_residual_n(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(xs), SWIGTYPE_p_float.getCPtr(residuals), SWIGTYPE_p_long_long.getCPtr(keys.data()), keys); } public DistanceComputer get_distance_computer() { long cPtr = swigfaissJNI.Index_get_distance_computer(swigCPtr, this); return (cPtr == 0) ? null : new DistanceComputer(cPtr, false); } public long sa_code_size() { return swigfaissJNI.Index_sa_code_size(swigCPtr, this); } public void sa_encode(long n, SWIGTYPE_p_float x, SWIGTYPE_p_unsigned_char bytes) { swigfaissJNI.Index_sa_encode(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_unsigned_char.getCPtr(bytes)); } public void sa_decode(long n, SWIGTYPE_p_unsigned_char bytes, SWIGTYPE_p_float x) { swigfaissJNI.Index_sa_decode(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(bytes), SWIGTYPE_p_float.getCPtr(x)); } public IndexIVF toIVF() { long cPtr = swigfaissJNI.Index_toIVF(swigCPtr, this); return (cPtr == 0) ? null : new IndexIVF(cPtr, false); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/Index2Layer.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class Index2Layer extends IndexFlatCodes { private transient long swigCPtr; protected Index2Layer(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.Index2Layer_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(Index2Layer obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_Index2Layer(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setQ1(Level1Quantizer value) { swigfaissJNI.Index2Layer_q1_set(swigCPtr, this, Level1Quantizer.getCPtr(value), value); } public Level1Quantizer getQ1() { long cPtr = swigfaissJNI.Index2Layer_q1_get(swigCPtr, this); return (cPtr == 0) ? null : new Level1Quantizer(cPtr, false); } public void setPq(ProductQuantizer value) { swigfaissJNI.Index2Layer_pq_set(swigCPtr, this, ProductQuantizer.getCPtr(value), value); } public ProductQuantizer getPq() { long cPtr = swigfaissJNI.Index2Layer_pq_get(swigCPtr, this); return (cPtr == 0) ? null : new ProductQuantizer(cPtr, false); } public void setCode_size_1(long value) { swigfaissJNI.Index2Layer_code_size_1_set(swigCPtr, this, value); } public long getCode_size_1() { return swigfaissJNI.Index2Layer_code_size_1_get(swigCPtr, this); } public void setCode_size_2(long value) { swigfaissJNI.Index2Layer_code_size_2_set(swigCPtr, this, value); } public long getCode_size_2() { return swigfaissJNI.Index2Layer_code_size_2_get(swigCPtr, this); } public Index2Layer(Index quantizer, long nlist, int M, int nbit, MetricType metric) { this(swigfaissJNI.new_Index2Layer__SWIG_0(Index.getCPtr(quantizer), quantizer, nlist, M, nbit, metric.swigValue()), true); } public Index2Layer(Index quantizer, long nlist, int M, int nbit) { this(swigfaissJNI.new_Index2Layer__SWIG_1(Index.getCPtr(quantizer), quantizer, nlist, M, nbit), true); } public Index2Layer(Index quantizer, long nlist, int M) { this(swigfaissJNI.new_Index2Layer__SWIG_2(Index.getCPtr(quantizer), quantizer, nlist, M), true); } public Index2Layer() { this(swigfaissJNI.new_Index2Layer__SWIG_3(), true); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.Index2Layer_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.Index2Layer_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public DistanceComputer get_distance_computer() { long cPtr = swigfaissJNI.Index2Layer_get_distance_computer(swigCPtr, this); return (cPtr == 0) ? null : new DistanceComputer(cPtr, false); } public void transfer_to_IVFPQ(IndexIVFPQ other) { swigfaissJNI.Index2Layer_transfer_to_IVFPQ(swigCPtr, this, IndexIVFPQ.getCPtr(other), other); } public void sa_encode(long n, SWIGTYPE_p_float x, SWIGTYPE_p_unsigned_char bytes) { swigfaissJNI.Index2Layer_sa_encode(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_unsigned_char.getCPtr(bytes)); } public void sa_decode(long n, SWIGTYPE_p_unsigned_char bytes, SWIGTYPE_p_float x) { swigfaissJNI.Index2Layer_sa_decode(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(bytes), SWIGTYPE_p_float.getCPtr(x)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexBinary.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexBinary { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected IndexBinary(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(IndexBinary obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexBinary(swigCPtr); } swigCPtr = 0; } } public void setD(int value) { swigfaissJNI.IndexBinary_d_set(swigCPtr, this, value); } public int getD() { return swigfaissJNI.IndexBinary_d_get(swigCPtr, this); } public void setCode_size(int value) { swigfaissJNI.IndexBinary_code_size_set(swigCPtr, this, value); } public int getCode_size() { return swigfaissJNI.IndexBinary_code_size_get(swigCPtr, this); } public void setNtotal(long value) { swigfaissJNI.IndexBinary_ntotal_set(swigCPtr, this, value); } public long getNtotal() { return swigfaissJNI.IndexBinary_ntotal_get(swigCPtr, this); } public void setVerbose(boolean value) { swigfaissJNI.IndexBinary_verbose_set(swigCPtr, this, value); } public boolean getVerbose() { return swigfaissJNI.IndexBinary_verbose_get(swigCPtr, this); } public void setIs_trained(boolean value) { swigfaissJNI.IndexBinary_is_trained_set(swigCPtr, this, value); } public boolean getIs_trained() { return swigfaissJNI.IndexBinary_is_trained_get(swigCPtr, this); } public void setMetric_type(MetricType value) { swigfaissJNI.IndexBinary_metric_type_set(swigCPtr, this, value.swigValue()); } public MetricType getMetric_type() { return MetricType.swigToEnum(swigfaissJNI.IndexBinary_metric_type_get(swigCPtr, this)); } public void train(long n, SWIGTYPE_p_unsigned_char x) { swigfaissJNI.IndexBinary_train(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x)); } public void add(long n, SWIGTYPE_p_unsigned_char x) { swigfaissJNI.IndexBinary_add(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x)); } public void add_with_ids(long n, SWIGTYPE_p_unsigned_char x, LongVector xids) { swigfaissJNI.IndexBinary_add_with_ids(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(xids.data()), xids); } public void search(long n, SWIGTYPE_p_unsigned_char x, long k, SWIGTYPE_p_int distances, LongVector labels) { swigfaissJNI.IndexBinary_search(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), k, SWIGTYPE_p_int.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void range_search(long n, SWIGTYPE_p_unsigned_char x, int radius, RangeSearchResult result) { swigfaissJNI.IndexBinary_range_search(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), radius, RangeSearchResult.getCPtr(result), result); } public void assign(long n, SWIGTYPE_p_unsigned_char x, LongVector labels, long k) { swigfaissJNI.IndexBinary_assign__SWIG_0(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, k); } public void assign(long n, SWIGTYPE_p_unsigned_char x, LongVector labels) { swigfaissJNI.IndexBinary_assign__SWIG_1(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void reset() { swigfaissJNI.IndexBinary_reset(swigCPtr, this); } public long remove_ids(IDSelector sel) { return swigfaissJNI.IndexBinary_remove_ids(swigCPtr, this, IDSelector.getCPtr(sel), sel); } public void reconstruct(long key, SWIGTYPE_p_unsigned_char recons) { swigfaissJNI.IndexBinary_reconstruct(swigCPtr, this, key, SWIGTYPE_p_unsigned_char.getCPtr(recons)); } public void reconstruct_n(long i0, long ni, SWIGTYPE_p_unsigned_char recons) { swigfaissJNI.IndexBinary_reconstruct_n(swigCPtr, this, i0, ni, SWIGTYPE_p_unsigned_char.getCPtr(recons)); } public void search_and_reconstruct(long n, SWIGTYPE_p_unsigned_char x, long k, SWIGTYPE_p_int distances, LongVector labels, SWIGTYPE_p_unsigned_char recons) { swigfaissJNI.IndexBinary_search_and_reconstruct(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), k, SWIGTYPE_p_int.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, SWIGTYPE_p_unsigned_char.getCPtr(recons)); } public void display() { swigfaissJNI.IndexBinary_display(swigCPtr, this); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexBinaryFlat.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexBinaryFlat extends IndexBinary { private transient long swigCPtr; protected IndexBinaryFlat(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexBinaryFlat_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexBinaryFlat obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexBinaryFlat(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setXb(ByteVector value) { swigfaissJNI.IndexBinaryFlat_xb_set(swigCPtr, this, ByteVector.getCPtr(value), value); } public ByteVector getXb() { long cPtr = swigfaissJNI.IndexBinaryFlat_xb_get(swigCPtr, this); return (cPtr == 0) ? null : new ByteVector(cPtr, false); } public void setUse_heap(boolean value) { swigfaissJNI.IndexBinaryFlat_use_heap_set(swigCPtr, this, value); } public boolean getUse_heap() { return swigfaissJNI.IndexBinaryFlat_use_heap_get(swigCPtr, this); } public void setQuery_batch_size(long value) { swigfaissJNI.IndexBinaryFlat_query_batch_size_set(swigCPtr, this, value); } public long getQuery_batch_size() { return swigfaissJNI.IndexBinaryFlat_query_batch_size_get(swigCPtr, this); } public IndexBinaryFlat(long d) { this(swigfaissJNI.new_IndexBinaryFlat__SWIG_0(d), true); } public void add(long n, SWIGTYPE_p_unsigned_char x) { swigfaissJNI.IndexBinaryFlat_add(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x)); } public void reset() { swigfaissJNI.IndexBinaryFlat_reset(swigCPtr, this); } public void search(long n, SWIGTYPE_p_unsigned_char x, long k, SWIGTYPE_p_int distances, LongVector labels) { swigfaissJNI.IndexBinaryFlat_search(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), k, SWIGTYPE_p_int.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void range_search(long n, SWIGTYPE_p_unsigned_char x, int radius, RangeSearchResult result) { swigfaissJNI.IndexBinaryFlat_range_search(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), radius, RangeSearchResult.getCPtr(result), result); } public void reconstruct(long key, SWIGTYPE_p_unsigned_char recons) { swigfaissJNI.IndexBinaryFlat_reconstruct(swigCPtr, this, key, SWIGTYPE_p_unsigned_char.getCPtr(recons)); } public long remove_ids(IDSelector sel) { return swigfaissJNI.IndexBinaryFlat_remove_ids(swigCPtr, this, IDSelector.getCPtr(sel), sel); } public IndexBinaryFlat() { this(swigfaissJNI.new_IndexBinaryFlat__SWIG_1(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexBinaryFromFloat.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexBinaryFromFloat extends IndexBinary { private transient long swigCPtr; protected IndexBinaryFromFloat(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexBinaryFromFloat_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexBinaryFromFloat obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexBinaryFromFloat(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setIndex(Index value) { swigfaissJNI.IndexBinaryFromFloat_index_set(swigCPtr, this, Index.getCPtr(value), value); } public Index getIndex() { long cPtr = swigfaissJNI.IndexBinaryFromFloat_index_get(swigCPtr, this); return (cPtr == 0) ? null : new Index(cPtr, false); } public void setOwn_fields(boolean value) { swigfaissJNI.IndexBinaryFromFloat_own_fields_set(swigCPtr, this, value); } public boolean getOwn_fields() { return swigfaissJNI.IndexBinaryFromFloat_own_fields_get(swigCPtr, this); } public IndexBinaryFromFloat() { this(swigfaissJNI.new_IndexBinaryFromFloat__SWIG_0(), true); } public IndexBinaryFromFloat(Index index) { this(swigfaissJNI.new_IndexBinaryFromFloat__SWIG_1(Index.getCPtr(index), index), true); } public void add(long n, SWIGTYPE_p_unsigned_char x) { swigfaissJNI.IndexBinaryFromFloat_add(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x)); } public void reset() { swigfaissJNI.IndexBinaryFromFloat_reset(swigCPtr, this); } public void search(long n, SWIGTYPE_p_unsigned_char x, long k, SWIGTYPE_p_int distances, LongVector labels) { swigfaissJNI.IndexBinaryFromFloat_search(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), k, SWIGTYPE_p_int.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void train(long n, SWIGTYPE_p_unsigned_char x) { swigfaissJNI.IndexBinaryFromFloat_train(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexBinaryHNSW.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexBinaryHNSW extends IndexBinary { private transient long swigCPtr; protected IndexBinaryHNSW(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexBinaryHNSW_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexBinaryHNSW obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexBinaryHNSW(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setHnsw(HNSW value) { swigfaissJNI.IndexBinaryHNSW_hnsw_set(swigCPtr, this, HNSW.getCPtr(value), value); } public HNSW getHnsw() { long cPtr = swigfaissJNI.IndexBinaryHNSW_hnsw_get(swigCPtr, this); return (cPtr == 0) ? null : new HNSW(cPtr, false); } public void setOwn_fields(boolean value) { swigfaissJNI.IndexBinaryHNSW_own_fields_set(swigCPtr, this, value); } public boolean getOwn_fields() { return swigfaissJNI.IndexBinaryHNSW_own_fields_get(swigCPtr, this); } public void setStorage(IndexBinary value) { swigfaissJNI.IndexBinaryHNSW_storage_set(swigCPtr, this, IndexBinary.getCPtr(value), value); } public IndexBinary getStorage() { long cPtr = swigfaissJNI.IndexBinaryHNSW_storage_get(swigCPtr, this); return (cPtr == 0) ? null : new IndexBinary(cPtr, false); } public IndexBinaryHNSW() { this(swigfaissJNI.new_IndexBinaryHNSW__SWIG_0(), true); } public IndexBinaryHNSW(int d, int M) { this(swigfaissJNI.new_IndexBinaryHNSW__SWIG_1(d, M), true); } public IndexBinaryHNSW(int d) { this(swigfaissJNI.new_IndexBinaryHNSW__SWIG_2(d), true); } public IndexBinaryHNSW(IndexBinary storage, int M) { this(swigfaissJNI.new_IndexBinaryHNSW__SWIG_3(IndexBinary.getCPtr(storage), storage, M), true); } public IndexBinaryHNSW(IndexBinary storage) { this(swigfaissJNI.new_IndexBinaryHNSW__SWIG_4(IndexBinary.getCPtr(storage), storage), true); } public DistanceComputer get_distance_computer() { long cPtr = swigfaissJNI.IndexBinaryHNSW_get_distance_computer(swigCPtr, this); return (cPtr == 0) ? null : new DistanceComputer(cPtr, false); } public void add(long n, SWIGTYPE_p_unsigned_char x) { swigfaissJNI.IndexBinaryHNSW_add(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x)); } public void train(long n, SWIGTYPE_p_unsigned_char x) { swigfaissJNI.IndexBinaryHNSW_train(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x)); } public void search(long n, SWIGTYPE_p_unsigned_char x, long k, SWIGTYPE_p_int distances, LongVector labels) { swigfaissJNI.IndexBinaryHNSW_search(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), k, SWIGTYPE_p_int.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void reconstruct(long key, SWIGTYPE_p_unsigned_char recons) { swigfaissJNI.IndexBinaryHNSW_reconstruct(swigCPtr, this, key, SWIGTYPE_p_unsigned_char.getCPtr(recons)); } public void reset() { swigfaissJNI.IndexBinaryHNSW_reset(swigCPtr, this); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexBinaryIVF.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexBinaryIVF extends IndexBinary { private transient long swigCPtr; protected IndexBinaryIVF(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexBinaryIVF_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexBinaryIVF obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexBinaryIVF(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setInvlists(InvertedLists value) { swigfaissJNI.IndexBinaryIVF_invlists_set(swigCPtr, this, InvertedLists.getCPtr(value), value); } public InvertedLists getInvlists() { long cPtr = swigfaissJNI.IndexBinaryIVF_invlists_get(swigCPtr, this); return (cPtr == 0) ? null : new InvertedLists(cPtr, false); } public void setOwn_invlists(boolean value) { swigfaissJNI.IndexBinaryIVF_own_invlists_set(swigCPtr, this, value); } public boolean getOwn_invlists() { return swigfaissJNI.IndexBinaryIVF_own_invlists_get(swigCPtr, this); } public void setNprobe(long value) { swigfaissJNI.IndexBinaryIVF_nprobe_set(swigCPtr, this, value); } public long getNprobe() { return swigfaissJNI.IndexBinaryIVF_nprobe_get(swigCPtr, this); } public void setMax_codes(long value) { swigfaissJNI.IndexBinaryIVF_max_codes_set(swigCPtr, this, value); } public long getMax_codes() { return swigfaissJNI.IndexBinaryIVF_max_codes_get(swigCPtr, this); } public void setUse_heap(boolean value) { swigfaissJNI.IndexBinaryIVF_use_heap_set(swigCPtr, this, value); } public boolean getUse_heap() { return swigfaissJNI.IndexBinaryIVF_use_heap_get(swigCPtr, this); } public void setDirect_map(SWIGTYPE_p_DirectMap value) { swigfaissJNI.IndexBinaryIVF_direct_map_set(swigCPtr, this, SWIGTYPE_p_DirectMap.getCPtr(value)); } public SWIGTYPE_p_DirectMap getDirect_map() { return new SWIGTYPE_p_DirectMap(swigfaissJNI.IndexBinaryIVF_direct_map_get(swigCPtr, this), true); } public void setQuantizer(IndexBinary value) { swigfaissJNI.IndexBinaryIVF_quantizer_set(swigCPtr, this, IndexBinary.getCPtr(value), value); } public IndexBinary getQuantizer() { long cPtr = swigfaissJNI.IndexBinaryIVF_quantizer_get(swigCPtr, this); return (cPtr == 0) ? null : new IndexBinary(cPtr, false); } public void setNlist(long value) { swigfaissJNI.IndexBinaryIVF_nlist_set(swigCPtr, this, value); } public long getNlist() { return swigfaissJNI.IndexBinaryIVF_nlist_get(swigCPtr, this); } public void setOwn_fields(boolean value) { swigfaissJNI.IndexBinaryIVF_own_fields_set(swigCPtr, this, value); } public boolean getOwn_fields() { return swigfaissJNI.IndexBinaryIVF_own_fields_get(swigCPtr, this); } public void setCp(ClusteringParameters value) { swigfaissJNI.IndexBinaryIVF_cp_set(swigCPtr, this, ClusteringParameters.getCPtr(value), value); } public ClusteringParameters getCp() { long cPtr = swigfaissJNI.IndexBinaryIVF_cp_get(swigCPtr, this); return (cPtr == 0) ? null : new ClusteringParameters(cPtr, false); } public void setClustering_index(Index value) { swigfaissJNI.IndexBinaryIVF_clustering_index_set(swigCPtr, this, Index.getCPtr(value), value); } public Index getClustering_index() { long cPtr = swigfaissJNI.IndexBinaryIVF_clustering_index_get(swigCPtr, this); return (cPtr == 0) ? null : new Index(cPtr, false); } public IndexBinaryIVF(IndexBinary quantizer, long d, long nlist) { this(swigfaissJNI.new_IndexBinaryIVF__SWIG_0(IndexBinary.getCPtr(quantizer), quantizer, d, nlist), true); } public IndexBinaryIVF() { this(swigfaissJNI.new_IndexBinaryIVF__SWIG_1(), true); } public void reset() { swigfaissJNI.IndexBinaryIVF_reset(swigCPtr, this); } public void train(long n, SWIGTYPE_p_unsigned_char x) { swigfaissJNI.IndexBinaryIVF_train(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x)); } public void add(long n, SWIGTYPE_p_unsigned_char x) { swigfaissJNI.IndexBinaryIVF_add(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x)); } public void add_with_ids(long n, SWIGTYPE_p_unsigned_char x, LongVector xids) { swigfaissJNI.IndexBinaryIVF_add_with_ids(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(xids.data()), xids); } public void add_core(long n, SWIGTYPE_p_unsigned_char x, LongVector xids, LongVector precomputed_idx) { swigfaissJNI.IndexBinaryIVF_add_core(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(xids.data()), xids, SWIGTYPE_p_long_long.getCPtr(precomputed_idx.data()), precomputed_idx); } public void search_preassigned(long n, SWIGTYPE_p_unsigned_char x, long k, LongVector assign, SWIGTYPE_p_int centroid_dis, SWIGTYPE_p_int distances, LongVector labels, boolean store_pairs, IVFSearchParameters params) { swigfaissJNI.IndexBinaryIVF_search_preassigned__SWIG_0(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), k, SWIGTYPE_p_long_long.getCPtr(assign.data()), assign, SWIGTYPE_p_int.getCPtr(centroid_dis), SWIGTYPE_p_int.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, store_pairs, IVFSearchParameters.getCPtr(params), params); } public void search_preassigned(long n, SWIGTYPE_p_unsigned_char x, long k, LongVector assign, SWIGTYPE_p_int centroid_dis, SWIGTYPE_p_int distances, LongVector labels, boolean store_pairs) { swigfaissJNI.IndexBinaryIVF_search_preassigned__SWIG_1(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), k, SWIGTYPE_p_long_long.getCPtr(assign.data()), assign, SWIGTYPE_p_int.getCPtr(centroid_dis), SWIGTYPE_p_int.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, store_pairs); } public SWIGTYPE_p_faiss__BinaryInvertedListScanner get_InvertedListScanner(boolean store_pairs) { long cPtr = swigfaissJNI.IndexBinaryIVF_get_InvertedListScanner__SWIG_0(swigCPtr, this, store_pairs); return (cPtr == 0) ? null : new SWIGTYPE_p_faiss__BinaryInvertedListScanner(cPtr, false); } public SWIGTYPE_p_faiss__BinaryInvertedListScanner get_InvertedListScanner() { long cPtr = swigfaissJNI.IndexBinaryIVF_get_InvertedListScanner__SWIG_1(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_faiss__BinaryInvertedListScanner(cPtr, false); } public void search(long n, SWIGTYPE_p_unsigned_char x, long k, SWIGTYPE_p_int distances, LongVector labels) { swigfaissJNI.IndexBinaryIVF_search(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), k, SWIGTYPE_p_int.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void range_search(long n, SWIGTYPE_p_unsigned_char x, int radius, RangeSearchResult result) { swigfaissJNI.IndexBinaryIVF_range_search(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), radius, RangeSearchResult.getCPtr(result), result); } public void range_search_preassigned(long n, SWIGTYPE_p_unsigned_char x, int radius, LongVector assign, SWIGTYPE_p_int centroid_dis, RangeSearchResult result) { swigfaissJNI.IndexBinaryIVF_range_search_preassigned(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), radius, SWIGTYPE_p_long_long.getCPtr(assign.data()), assign, SWIGTYPE_p_int.getCPtr(centroid_dis), RangeSearchResult.getCPtr(result), result); } public void reconstruct(long key, SWIGTYPE_p_unsigned_char recons) { swigfaissJNI.IndexBinaryIVF_reconstruct(swigCPtr, this, key, SWIGTYPE_p_unsigned_char.getCPtr(recons)); } public void reconstruct_n(long i0, long ni, SWIGTYPE_p_unsigned_char recons) { swigfaissJNI.IndexBinaryIVF_reconstruct_n(swigCPtr, this, i0, ni, SWIGTYPE_p_unsigned_char.getCPtr(recons)); } public void search_and_reconstruct(long n, SWIGTYPE_p_unsigned_char x, long k, SWIGTYPE_p_int distances, LongVector labels, SWIGTYPE_p_unsigned_char recons) { swigfaissJNI.IndexBinaryIVF_search_and_reconstruct(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(x), k, SWIGTYPE_p_int.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, SWIGTYPE_p_unsigned_char.getCPtr(recons)); } public void reconstruct_from_offset(long list_no, long offset, SWIGTYPE_p_unsigned_char recons) { swigfaissJNI.IndexBinaryIVF_reconstruct_from_offset(swigCPtr, this, list_no, offset, SWIGTYPE_p_unsigned_char.getCPtr(recons)); } public long remove_ids(IDSelector sel) { return swigfaissJNI.IndexBinaryIVF_remove_ids(swigCPtr, this, IDSelector.getCPtr(sel), sel); } public void merge_from(IndexBinaryIVF other, long add_id) { swigfaissJNI.IndexBinaryIVF_merge_from(swigCPtr, this, IndexBinaryIVF.getCPtr(other), other, add_id); } public long get_list_size(long list_no) { return swigfaissJNI.IndexBinaryIVF_get_list_size(swigCPtr, this, list_no); } public void make_direct_map(boolean new_maintain_direct_map) { swigfaissJNI.IndexBinaryIVF_make_direct_map__SWIG_0(swigCPtr, this, new_maintain_direct_map); } public void make_direct_map() { swigfaissJNI.IndexBinaryIVF_make_direct_map__SWIG_1(swigCPtr, this); } public void set_direct_map_type(SWIGTYPE_p_DirectMap__Type type) { swigfaissJNI.IndexBinaryIVF_set_direct_map_type(swigCPtr, this, SWIGTYPE_p_DirectMap__Type.getCPtr(type)); } public void replace_invlists(InvertedLists il, boolean own) { swigfaissJNI.IndexBinaryIVF_replace_invlists__SWIG_0(swigCPtr, this, InvertedLists.getCPtr(il), il, own); } public void replace_invlists(InvertedLists il) { swigfaissJNI.IndexBinaryIVF_replace_invlists__SWIG_1(swigCPtr, this, InvertedLists.getCPtr(il), il); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexFlat.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexFlat extends IndexFlatCodes { private transient long swigCPtr; protected IndexFlat(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexFlat_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexFlat obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexFlat(swigCPtr); } swigCPtr = 0; } super.delete(); } public IndexFlat(long d, MetricType metric) { this(swigfaissJNI.new_IndexFlat__SWIG_0(d, metric.swigValue()), true); } public IndexFlat(long d) { this(swigfaissJNI.new_IndexFlat__SWIG_1(d), true); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.IndexFlat_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void range_search(long n, SWIGTYPE_p_float x, float radius, RangeSearchResult result) { swigfaissJNI.IndexFlat_range_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), radius, RangeSearchResult.getCPtr(result), result); } public void reconstruct(long key, SWIGTYPE_p_float recons) { swigfaissJNI.IndexFlat_reconstruct(swigCPtr, this, key, SWIGTYPE_p_float.getCPtr(recons)); } public void compute_distance_subset(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.IndexFlat_compute_distance_subset(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public SWIGTYPE_p_float get_xb() { long cPtr = swigfaissJNI.IndexFlat_get_xb__SWIG_0(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } public IndexFlat() { this(swigfaissJNI.new_IndexFlat__SWIG_2(), true); } public DistanceComputer get_distance_computer() { long cPtr = swigfaissJNI.IndexFlat_get_distance_computer(swigCPtr, this); return (cPtr == 0) ? null : new DistanceComputer(cPtr, false); } public void sa_encode(long n, SWIGTYPE_p_float x, SWIGTYPE_p_unsigned_char bytes) { swigfaissJNI.IndexFlat_sa_encode(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_unsigned_char.getCPtr(bytes)); } public void sa_decode(long n, SWIGTYPE_p_unsigned_char bytes, SWIGTYPE_p_float x) { swigfaissJNI.IndexFlat_sa_decode(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(bytes), SWIGTYPE_p_float.getCPtr(x)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexFlat1D.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexFlat1D extends IndexFlatL2 { private transient long swigCPtr; protected IndexFlat1D(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexFlat1D_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexFlat1D obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexFlat1D(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setContinuous_update(boolean value) { swigfaissJNI.IndexFlat1D_continuous_update_set(swigCPtr, this, value); } public boolean getContinuous_update() { return swigfaissJNI.IndexFlat1D_continuous_update_get(swigCPtr, this); } public void setPerm(SWIGTYPE_p_std__vectorT_int64_t_t value) { swigfaissJNI.IndexFlat1D_perm_set(swigCPtr, this, SWIGTYPE_p_std__vectorT_int64_t_t.getCPtr(value)); } public SWIGTYPE_p_std__vectorT_int64_t_t getPerm() { long cPtr = swigfaissJNI.IndexFlat1D_perm_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_int64_t_t(cPtr, false); } public IndexFlat1D(boolean continuous_update) { this(swigfaissJNI.new_IndexFlat1D__SWIG_0(continuous_update), true); } public IndexFlat1D() { this(swigfaissJNI.new_IndexFlat1D__SWIG_1(), true); } public void update_permutation() { swigfaissJNI.IndexFlat1D_update_permutation(swigCPtr, this); } public void add(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexFlat1D_add(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void reset() { swigfaissJNI.IndexFlat1D_reset(swigCPtr, this); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.IndexFlat1D_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexFlatCodes.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexFlatCodes extends Index { private transient long swigCPtr; protected IndexFlatCodes(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexFlatCodes_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexFlatCodes obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexFlatCodes(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setCode_size(long value) { swigfaissJNI.IndexFlatCodes_code_size_set(swigCPtr, this, value); } public long getCode_size() { return swigfaissJNI.IndexFlatCodes_code_size_get(swigCPtr, this); } public void setCodes(ByteVector value) { swigfaissJNI.IndexFlatCodes_codes_set(swigCPtr, this, ByteVector.getCPtr(value), value); } public ByteVector getCodes() { long cPtr = swigfaissJNI.IndexFlatCodes_codes_get(swigCPtr, this); return (cPtr == 0) ? null : new ByteVector(cPtr, false); } public void add(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexFlatCodes_add(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void reset() { swigfaissJNI.IndexFlatCodes_reset(swigCPtr, this); } public void reconstruct_n(long i0, long ni, SWIGTYPE_p_float recons) { swigfaissJNI.IndexFlatCodes_reconstruct_n(swigCPtr, this, i0, ni, SWIGTYPE_p_float.getCPtr(recons)); } public void reconstruct(long key, SWIGTYPE_p_float recons) { swigfaissJNI.IndexFlatCodes_reconstruct(swigCPtr, this, key, SWIGTYPE_p_float.getCPtr(recons)); } public long sa_code_size() { return swigfaissJNI.IndexFlatCodes_sa_code_size(swigCPtr, this); } public long remove_ids(IDSelector sel) { return swigfaissJNI.IndexFlatCodes_remove_ids(swigCPtr, this, IDSelector.getCPtr(sel), sel); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexFlatIP.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexFlatIP extends IndexFlat { private transient long swigCPtr; protected IndexFlatIP(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexFlatIP_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexFlatIP obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexFlatIP(swigCPtr); } swigCPtr = 0; } super.delete(); } public IndexFlatIP(long d) { this(swigfaissJNI.new_IndexFlatIP__SWIG_0(d), true); } public IndexFlatIP() { this(swigfaissJNI.new_IndexFlatIP__SWIG_1(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexFlatL2.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexFlatL2 extends IndexFlat { private transient long swigCPtr; protected IndexFlatL2(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexFlatL2_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexFlatL2 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexFlatL2(swigCPtr); } swigCPtr = 0; } super.delete(); } public IndexFlatL2(long d) { this(swigfaissJNI.new_IndexFlatL2__SWIG_0(d), true); } public IndexFlatL2() { this(swigfaissJNI.new_IndexFlatL2__SWIG_1(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexHNSW.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexHNSW extends Index { private transient long swigCPtr; protected IndexHNSW(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexHNSW_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexHNSW obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexHNSW(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setHnsw(HNSW value) { swigfaissJNI.IndexHNSW_hnsw_set(swigCPtr, this, HNSW.getCPtr(value), value); } public HNSW getHnsw() { long cPtr = swigfaissJNI.IndexHNSW_hnsw_get(swigCPtr, this); return (cPtr == 0) ? null : new HNSW(cPtr, false); } public void setOwn_fields(boolean value) { swigfaissJNI.IndexHNSW_own_fields_set(swigCPtr, this, value); } public boolean getOwn_fields() { return swigfaissJNI.IndexHNSW_own_fields_get(swigCPtr, this); } public void setStorage(Index value) { swigfaissJNI.IndexHNSW_storage_set(swigCPtr, this, Index.getCPtr(value), value); } public Index getStorage() { long cPtr = swigfaissJNI.IndexHNSW_storage_get(swigCPtr, this); return (cPtr == 0) ? null : new Index(cPtr, false); } public void setReconstruct_from_neighbors(ReconstructFromNeighbors value) { swigfaissJNI.IndexHNSW_reconstruct_from_neighbors_set(swigCPtr, this, ReconstructFromNeighbors.getCPtr(value), value); } public ReconstructFromNeighbors getReconstruct_from_neighbors() { long cPtr = swigfaissJNI.IndexHNSW_reconstruct_from_neighbors_get(swigCPtr, this); return (cPtr == 0) ? null : new ReconstructFromNeighbors(cPtr, false); } public IndexHNSW(int d, int M, MetricType metric) { this(swigfaissJNI.new_IndexHNSW__SWIG_0(d, M, metric.swigValue()), true); } public IndexHNSW(int d, int M) { this(swigfaissJNI.new_IndexHNSW__SWIG_1(d, M), true); } public IndexHNSW(int d) { this(swigfaissJNI.new_IndexHNSW__SWIG_2(d), true); } public IndexHNSW() { this(swigfaissJNI.new_IndexHNSW__SWIG_3(), true); } public IndexHNSW(Index storage, int M) { this(swigfaissJNI.new_IndexHNSW__SWIG_4(Index.getCPtr(storage), storage, M), true); } public IndexHNSW(Index storage) { this(swigfaissJNI.new_IndexHNSW__SWIG_5(Index.getCPtr(storage), storage), true); } public void add(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexHNSW_add(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexHNSW_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.IndexHNSW_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void reconstruct(long key, SWIGTYPE_p_float recons) { swigfaissJNI.IndexHNSW_reconstruct(swigCPtr, this, key, SWIGTYPE_p_float.getCPtr(recons)); } public void reset() { swigfaissJNI.IndexHNSW_reset(swigCPtr, this); } public void shrink_level_0_neighbors(int size) { swigfaissJNI.IndexHNSW_shrink_level_0_neighbors(swigCPtr, this, size); } public void search_level_0(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_int nearest, SWIGTYPE_p_float nearest_d, SWIGTYPE_p_float distances, LongVector labels, int nprobe, int search_type) { swigfaissJNI.IndexHNSW_search_level_0__SWIG_0(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_int.getCPtr(nearest), SWIGTYPE_p_float.getCPtr(nearest_d), SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, nprobe, search_type); } public void search_level_0(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_int nearest, SWIGTYPE_p_float nearest_d, SWIGTYPE_p_float distances, LongVector labels, int nprobe) { swigfaissJNI.IndexHNSW_search_level_0__SWIG_1(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_int.getCPtr(nearest), SWIGTYPE_p_float.getCPtr(nearest_d), SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, nprobe); } public void search_level_0(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_int nearest, SWIGTYPE_p_float nearest_d, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.IndexHNSW_search_level_0__SWIG_2(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_int.getCPtr(nearest), SWIGTYPE_p_float.getCPtr(nearest_d), SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void init_level_0_from_knngraph(int k, SWIGTYPE_p_float D, LongVector I) { swigfaissJNI.IndexHNSW_init_level_0_from_knngraph(swigCPtr, this, k, SWIGTYPE_p_float.getCPtr(D), SWIGTYPE_p_long_long.getCPtr(I.data()), I); } public void init_level_0_from_entry_points(int npt, SWIGTYPE_p_int points, SWIGTYPE_p_int nearests) { swigfaissJNI.IndexHNSW_init_level_0_from_entry_points(swigCPtr, this, npt, SWIGTYPE_p_int.getCPtr(points), SWIGTYPE_p_int.getCPtr(nearests)); } public void reorder_links() { swigfaissJNI.IndexHNSW_reorder_links(swigCPtr, this); } public void link_singletons() { swigfaissJNI.IndexHNSW_link_singletons(swigCPtr, this); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexHNSW2Level.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexHNSW2Level extends IndexHNSW { private transient long swigCPtr; protected IndexHNSW2Level(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexHNSW2Level_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexHNSW2Level obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexHNSW2Level(swigCPtr); } swigCPtr = 0; } super.delete(); } public IndexHNSW2Level() { this(swigfaissJNI.new_IndexHNSW2Level__SWIG_0(), true); } public IndexHNSW2Level(Index quantizer, long nlist, int m_pq, int M) { this(swigfaissJNI.new_IndexHNSW2Level__SWIG_1(Index.getCPtr(quantizer), quantizer, nlist, m_pq, M), true); } public void flip_to_ivf() { swigfaissJNI.IndexHNSW2Level_flip_to_ivf(swigCPtr, this); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.IndexHNSW2Level_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexHNSWFlat.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexHNSWFlat extends IndexHNSW { private transient long swigCPtr; protected IndexHNSWFlat(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexHNSWFlat_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexHNSWFlat obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexHNSWFlat(swigCPtr); } swigCPtr = 0; } super.delete(); } public IndexHNSWFlat() { this(swigfaissJNI.new_IndexHNSWFlat__SWIG_0(), true); } public IndexHNSWFlat(int d, int M, MetricType metric) { this(swigfaissJNI.new_IndexHNSWFlat__SWIG_1(d, M, metric.swigValue()), true); } public IndexHNSWFlat(int d, int M) { this(swigfaissJNI.new_IndexHNSWFlat__SWIG_2(d, M), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexHNSWPQ.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexHNSWPQ extends IndexHNSW { private transient long swigCPtr; protected IndexHNSWPQ(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexHNSWPQ_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexHNSWPQ obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexHNSWPQ(swigCPtr); } swigCPtr = 0; } super.delete(); } public IndexHNSWPQ() { this(swigfaissJNI.new_IndexHNSWPQ__SWIG_0(), true); } public IndexHNSWPQ(int d, int pq_m, int M) { this(swigfaissJNI.new_IndexHNSWPQ__SWIG_1(d, pq_m, M), true); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexHNSWPQ_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexHNSWSQ.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexHNSWSQ extends IndexHNSW { private transient long swigCPtr; protected IndexHNSWSQ(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexHNSWSQ_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexHNSWSQ obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexHNSWSQ(swigCPtr); } swigCPtr = 0; } super.delete(); } public IndexHNSWSQ() { this(swigfaissJNI.new_IndexHNSWSQ__SWIG_0(), true); } public IndexHNSWSQ(int d, SWIGTYPE_p_ScalarQuantizer__QuantizerType qtype, int M, MetricType metric) { this(swigfaissJNI.new_IndexHNSWSQ__SWIG_1(d, SWIGTYPE_p_ScalarQuantizer__QuantizerType.getCPtr(qtype), M, metric.swigValue()), true); } public IndexHNSWSQ(int d, SWIGTYPE_p_ScalarQuantizer__QuantizerType qtype, int M) { this(swigfaissJNI.new_IndexHNSWSQ__SWIG_2(d, SWIGTYPE_p_ScalarQuantizer__QuantizerType.getCPtr(qtype), M), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexIDMap.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexIDMap extends Index { private transient long swigCPtr; protected IndexIDMap(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexIDMap_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexIDMap obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexIDMap(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setIndex(Index value) { swigfaissJNI.IndexIDMap_index_set(swigCPtr, this, Index.getCPtr(value), value); } public Index getIndex() { long cPtr = swigfaissJNI.IndexIDMap_index_get(swigCPtr, this); return (cPtr == 0) ? null : new Index(cPtr, false); } public void setOwn_fields(boolean value) { swigfaissJNI.IndexIDMap_own_fields_set(swigCPtr, this, value); } public boolean getOwn_fields() { return swigfaissJNI.IndexIDMap_own_fields_get(swigCPtr, this); } public void setId_map(SWIGTYPE_p_std__vectorT_int64_t_t value) { swigfaissJNI.IndexIDMap_id_map_set(swigCPtr, this, SWIGTYPE_p_std__vectorT_int64_t_t.getCPtr(value)); } public SWIGTYPE_p_std__vectorT_int64_t_t getId_map() { long cPtr = swigfaissJNI.IndexIDMap_id_map_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_int64_t_t(cPtr, false); } public IndexIDMap(Index index) { this(swigfaissJNI.new_IndexIDMap__SWIG_0(Index.getCPtr(index), index), true); } public void add_with_ids(long n, SWIGTYPE_p_float x, LongVector xids) { swigfaissJNI.IndexIDMap_add_with_ids(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(xids.data()), xids); } public void add(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexIDMap_add(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.IndexIDMap_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexIDMap_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void reset() { swigfaissJNI.IndexIDMap_reset(swigCPtr, this); } public long remove_ids(IDSelector sel) { return swigfaissJNI.IndexIDMap_remove_ids(swigCPtr, this, IDSelector.getCPtr(sel), sel); } public void range_search(long n, SWIGTYPE_p_float x, float radius, RangeSearchResult result) { swigfaissJNI.IndexIDMap_range_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), radius, RangeSearchResult.getCPtr(result), result); } public IndexIDMap() { this(swigfaissJNI.new_IndexIDMap__SWIG_1(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexIVF.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexIVF extends Index { private transient long swigCPtr; protected IndexIVF(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexIVF_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexIVF obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexIVF(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setInvlists(InvertedLists value) { swigfaissJNI.IndexIVF_invlists_set(swigCPtr, this, InvertedLists.getCPtr(value), value); } public InvertedLists getInvlists() { long cPtr = swigfaissJNI.IndexIVF_invlists_get(swigCPtr, this); return (cPtr == 0) ? null : new InvertedLists(cPtr, false); } public void setOwn_invlists(boolean value) { swigfaissJNI.IndexIVF_own_invlists_set(swigCPtr, this, value); } public boolean getOwn_invlists() { return swigfaissJNI.IndexIVF_own_invlists_get(swigCPtr, this); } public void setCode_size(long value) { swigfaissJNI.IndexIVF_code_size_set(swigCPtr, this, value); } public long getCode_size() { return swigfaissJNI.IndexIVF_code_size_get(swigCPtr, this); } public void setNprobe(long value) { swigfaissJNI.IndexIVF_nprobe_set(swigCPtr, this, value); } public long getNprobe() { return swigfaissJNI.IndexIVF_nprobe_get(swigCPtr, this); } public void setMax_codes(long value) { swigfaissJNI.IndexIVF_max_codes_set(swigCPtr, this, value); } public long getMax_codes() { return swigfaissJNI.IndexIVF_max_codes_get(swigCPtr, this); } public void setParallel_mode(int value) { swigfaissJNI.IndexIVF_parallel_mode_set(swigCPtr, this, value); } public int getParallel_mode() { return swigfaissJNI.IndexIVF_parallel_mode_get(swigCPtr, this); } public int getPARALLEL_MODE_NO_HEAP_INIT() { return swigfaissJNI.IndexIVF_PARALLEL_MODE_NO_HEAP_INIT_get(swigCPtr, this); } public void setDirect_map(SWIGTYPE_p_DirectMap value) { swigfaissJNI.IndexIVF_direct_map_set(swigCPtr, this, SWIGTYPE_p_DirectMap.getCPtr(value)); } public SWIGTYPE_p_DirectMap getDirect_map() { return new SWIGTYPE_p_DirectMap(swigfaissJNI.IndexIVF_direct_map_get(swigCPtr, this), true); } public void reset() { swigfaissJNI.IndexIVF_reset(swigCPtr, this); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexIVF_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void add(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexIVF_add(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void add_with_ids(long n, SWIGTYPE_p_float x, LongVector xids) { swigfaissJNI.IndexIVF_add_with_ids(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(xids.data()), xids); } public void add_core(long n, SWIGTYPE_p_float x, LongVector xids, LongVector precomputed_idx) { swigfaissJNI.IndexIVF_add_core(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(xids.data()), xids, SWIGTYPE_p_long_long.getCPtr(precomputed_idx.data()), precomputed_idx); } public void encode_vectors(long n, SWIGTYPE_p_float x, LongVector list_nos, SWIGTYPE_p_unsigned_char codes, boolean include_listno) { swigfaissJNI.IndexIVF_encode_vectors__SWIG_0(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(list_nos.data()), list_nos, SWIGTYPE_p_unsigned_char.getCPtr(codes), include_listno); } public void encode_vectors(long n, SWIGTYPE_p_float x, LongVector list_nos, SWIGTYPE_p_unsigned_char codes) { swigfaissJNI.IndexIVF_encode_vectors__SWIG_1(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(list_nos.data()), list_nos, SWIGTYPE_p_unsigned_char.getCPtr(codes)); } public void add_sa_codes(long n, SWIGTYPE_p_unsigned_char codes, LongVector xids) { swigfaissJNI.IndexIVF_add_sa_codes(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(codes), SWIGTYPE_p_long_long.getCPtr(xids.data()), xids); } public void train_residual(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexIVF_train_residual(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void search_preassigned(long n, SWIGTYPE_p_float x, long k, LongVector assign, SWIGTYPE_p_float centroid_dis, SWIGTYPE_p_float distances, LongVector labels, boolean store_pairs, IVFSearchParameters params, IndexIVFStats stats) { swigfaissJNI.IndexIVF_search_preassigned__SWIG_0(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_long_long.getCPtr(assign.data()), assign, SWIGTYPE_p_float.getCPtr(centroid_dis), SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, store_pairs, IVFSearchParameters.getCPtr(params), params, IndexIVFStats.getCPtr(stats), stats); } public void search_preassigned(long n, SWIGTYPE_p_float x, long k, LongVector assign, SWIGTYPE_p_float centroid_dis, SWIGTYPE_p_float distances, LongVector labels, boolean store_pairs, IVFSearchParameters params) { swigfaissJNI.IndexIVF_search_preassigned__SWIG_1(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_long_long.getCPtr(assign.data()), assign, SWIGTYPE_p_float.getCPtr(centroid_dis), SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, store_pairs, IVFSearchParameters.getCPtr(params), params); } public void search_preassigned(long n, SWIGTYPE_p_float x, long k, LongVector assign, SWIGTYPE_p_float centroid_dis, SWIGTYPE_p_float distances, LongVector labels, boolean store_pairs) { swigfaissJNI.IndexIVF_search_preassigned__SWIG_2(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_long_long.getCPtr(assign.data()), assign, SWIGTYPE_p_float.getCPtr(centroid_dis), SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, store_pairs); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.IndexIVF_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void range_search(long n, SWIGTYPE_p_float x, float radius, RangeSearchResult result) { swigfaissJNI.IndexIVF_range_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), radius, RangeSearchResult.getCPtr(result), result); } public void range_search_preassigned(long nx, SWIGTYPE_p_float x, float radius, LongVector keys, SWIGTYPE_p_float coarse_dis, RangeSearchResult result, boolean store_pairs, IVFSearchParameters params, IndexIVFStats stats) { swigfaissJNI.IndexIVF_range_search_preassigned__SWIG_0(swigCPtr, this, nx, SWIGTYPE_p_float.getCPtr(x), radius, SWIGTYPE_p_long_long.getCPtr(keys.data()), keys, SWIGTYPE_p_float.getCPtr(coarse_dis), RangeSearchResult.getCPtr(result), result, store_pairs, IVFSearchParameters.getCPtr(params), params, IndexIVFStats.getCPtr(stats), stats); } public void range_search_preassigned(long nx, SWIGTYPE_p_float x, float radius, LongVector keys, SWIGTYPE_p_float coarse_dis, RangeSearchResult result, boolean store_pairs, IVFSearchParameters params) { swigfaissJNI.IndexIVF_range_search_preassigned__SWIG_1(swigCPtr, this, nx, SWIGTYPE_p_float.getCPtr(x), radius, SWIGTYPE_p_long_long.getCPtr(keys.data()), keys, SWIGTYPE_p_float.getCPtr(coarse_dis), RangeSearchResult.getCPtr(result), result, store_pairs, IVFSearchParameters.getCPtr(params), params); } public void range_search_preassigned(long nx, SWIGTYPE_p_float x, float radius, LongVector keys, SWIGTYPE_p_float coarse_dis, RangeSearchResult result, boolean store_pairs) { swigfaissJNI.IndexIVF_range_search_preassigned__SWIG_2(swigCPtr, this, nx, SWIGTYPE_p_float.getCPtr(x), radius, SWIGTYPE_p_long_long.getCPtr(keys.data()), keys, SWIGTYPE_p_float.getCPtr(coarse_dis), RangeSearchResult.getCPtr(result), result, store_pairs); } public void range_search_preassigned(long nx, SWIGTYPE_p_float x, float radius, LongVector keys, SWIGTYPE_p_float coarse_dis, RangeSearchResult result) { swigfaissJNI.IndexIVF_range_search_preassigned__SWIG_3(swigCPtr, this, nx, SWIGTYPE_p_float.getCPtr(x), radius, SWIGTYPE_p_long_long.getCPtr(keys.data()), keys, SWIGTYPE_p_float.getCPtr(coarse_dis), RangeSearchResult.getCPtr(result), result); } public SWIGTYPE_p_faiss__InvertedListScanner get_InvertedListScanner(boolean store_pairs) { long cPtr = swigfaissJNI.IndexIVF_get_InvertedListScanner__SWIG_0(swigCPtr, this, store_pairs); return (cPtr == 0) ? null : new SWIGTYPE_p_faiss__InvertedListScanner(cPtr, false); } public SWIGTYPE_p_faiss__InvertedListScanner get_InvertedListScanner() { long cPtr = swigfaissJNI.IndexIVF_get_InvertedListScanner__SWIG_1(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_faiss__InvertedListScanner(cPtr, false); } public void reconstruct(long key, SWIGTYPE_p_float recons) { swigfaissJNI.IndexIVF_reconstruct(swigCPtr, this, key, SWIGTYPE_p_float.getCPtr(recons)); } public void update_vectors(int nv, LongVector idx, SWIGTYPE_p_float v) { swigfaissJNI.IndexIVF_update_vectors(swigCPtr, this, nv, SWIGTYPE_p_long_long.getCPtr(idx.data()), idx, SWIGTYPE_p_float.getCPtr(v)); } public void reconstruct_n(long i0, long ni, SWIGTYPE_p_float recons) { swigfaissJNI.IndexIVF_reconstruct_n(swigCPtr, this, i0, ni, SWIGTYPE_p_float.getCPtr(recons)); } public void search_and_reconstruct(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels, SWIGTYPE_p_float recons) { swigfaissJNI.IndexIVF_search_and_reconstruct(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, SWIGTYPE_p_float.getCPtr(recons)); } public void reconstruct_from_offset(long list_no, long offset, SWIGTYPE_p_float recons) { swigfaissJNI.IndexIVF_reconstruct_from_offset(swigCPtr, this, list_no, offset, SWIGTYPE_p_float.getCPtr(recons)); } public long remove_ids(IDSelector sel) { return swigfaissJNI.IndexIVF_remove_ids(swigCPtr, this, IDSelector.getCPtr(sel), sel); } public void check_compatible_for_merge(IndexIVF other) { swigfaissJNI.IndexIVF_check_compatible_for_merge(swigCPtr, this, IndexIVF.getCPtr(other), other); } public void merge_from(IndexIVF other, long add_id) { swigfaissJNI.IndexIVF_merge_from(swigCPtr, this, IndexIVF.getCPtr(other), other, add_id); } public void copy_subset_to(IndexIVF other, int subset_type, long a1, long a2) { swigfaissJNI.IndexIVF_copy_subset_to(swigCPtr, this, IndexIVF.getCPtr(other), other, subset_type, a1, a2); } public long get_list_size(long list_no) { return swigfaissJNI.IndexIVF_get_list_size(swigCPtr, this, list_no); } public void make_direct_map(boolean new_maintain_direct_map) { swigfaissJNI.IndexIVF_make_direct_map__SWIG_0(swigCPtr, this, new_maintain_direct_map); } public void make_direct_map() { swigfaissJNI.IndexIVF_make_direct_map__SWIG_1(swigCPtr, this); } public void set_direct_map_type(SWIGTYPE_p_DirectMap__Type type) { swigfaissJNI.IndexIVF_set_direct_map_type(swigCPtr, this, SWIGTYPE_p_DirectMap__Type.getCPtr(type)); } public void replace_invlists(InvertedLists il, boolean own) { swigfaissJNI.IndexIVF_replace_invlists__SWIG_0(swigCPtr, this, InvertedLists.getCPtr(il), il, own); } public void replace_invlists(InvertedLists il) { swigfaissJNI.IndexIVF_replace_invlists__SWIG_1(swigCPtr, this, InvertedLists.getCPtr(il), il); } public long sa_code_size() { return swigfaissJNI.IndexIVF_sa_code_size(swigCPtr, this); } public void sa_encode(long n, SWIGTYPE_p_float x, SWIGTYPE_p_unsigned_char bytes) { swigfaissJNI.IndexIVF_sa_encode(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_unsigned_char.getCPtr(bytes)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexIVFFlat.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexIVFFlat extends IndexIVF { private transient long swigCPtr; protected IndexIVFFlat(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexIVFFlat_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexIVFFlat obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexIVFFlat(swigCPtr); } swigCPtr = 0; } super.delete(); } public IndexIVFFlat(Index quantizer, long d, long nlist_, MetricType arg3) { this(swigfaissJNI.new_IndexIVFFlat__SWIG_0(Index.getCPtr(quantizer), quantizer, d, nlist_, arg3.swigValue()), true); } public IndexIVFFlat(Index quantizer, long d, long nlist_) { this(swigfaissJNI.new_IndexIVFFlat__SWIG_1(Index.getCPtr(quantizer), quantizer, d, nlist_), true); } public void add_core(long n, SWIGTYPE_p_float x, LongVector xids, LongVector precomputed_idx) { swigfaissJNI.IndexIVFFlat_add_core(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(xids.data()), xids, SWIGTYPE_p_long_long.getCPtr(precomputed_idx.data()), precomputed_idx); } public void encode_vectors(long n, SWIGTYPE_p_float x, LongVector list_nos, SWIGTYPE_p_unsigned_char codes, boolean include_listnos) { swigfaissJNI.IndexIVFFlat_encode_vectors__SWIG_0(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(list_nos.data()), list_nos, SWIGTYPE_p_unsigned_char.getCPtr(codes), include_listnos); } public void encode_vectors(long n, SWIGTYPE_p_float x, LongVector list_nos, SWIGTYPE_p_unsigned_char codes) { swigfaissJNI.IndexIVFFlat_encode_vectors__SWIG_1(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(list_nos.data()), list_nos, SWIGTYPE_p_unsigned_char.getCPtr(codes)); } public SWIGTYPE_p_faiss__InvertedListScanner get_InvertedListScanner(boolean store_pairs) { long cPtr = swigfaissJNI.IndexIVFFlat_get_InvertedListScanner(swigCPtr, this, store_pairs); return (cPtr == 0) ? null : new SWIGTYPE_p_faiss__InvertedListScanner(cPtr, false); } public void reconstruct_from_offset(long list_no, long offset, SWIGTYPE_p_float recons) { swigfaissJNI.IndexIVFFlat_reconstruct_from_offset(swigCPtr, this, list_no, offset, SWIGTYPE_p_float.getCPtr(recons)); } public void sa_decode(long n, SWIGTYPE_p_unsigned_char bytes, SWIGTYPE_p_float x) { swigfaissJNI.IndexIVFFlat_sa_decode(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(bytes), SWIGTYPE_p_float.getCPtr(x)); } public IndexIVFFlat() { this(swigfaissJNI.new_IndexIVFFlat__SWIG_2(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexIVFFlatDedup.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexIVFFlatDedup extends IndexIVFFlat { private transient long swigCPtr; protected IndexIVFFlatDedup(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexIVFFlatDedup_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexIVFFlatDedup obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexIVFFlatDedup(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setInstances(SWIGTYPE_p_std__unordered_multimapT_int64_t_int64_t_t value) { swigfaissJNI.IndexIVFFlatDedup_instances_set(swigCPtr, this, SWIGTYPE_p_std__unordered_multimapT_int64_t_int64_t_t.getCPtr(value)); } public SWIGTYPE_p_std__unordered_multimapT_int64_t_int64_t_t getInstances() { return new SWIGTYPE_p_std__unordered_multimapT_int64_t_int64_t_t(swigfaissJNI.IndexIVFFlatDedup_instances_get(swigCPtr, this), true); } public IndexIVFFlatDedup(Index quantizer, long d, long nlist_, MetricType arg3) { this(swigfaissJNI.new_IndexIVFFlatDedup__SWIG_0(Index.getCPtr(quantizer), quantizer, d, nlist_, arg3.swigValue()), true); } public IndexIVFFlatDedup(Index quantizer, long d, long nlist_) { this(swigfaissJNI.new_IndexIVFFlatDedup__SWIG_1(Index.getCPtr(quantizer), quantizer, d, nlist_), true); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexIVFFlatDedup_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void add_with_ids(long n, SWIGTYPE_p_float x, LongVector xids) { swigfaissJNI.IndexIVFFlatDedup_add_with_ids(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(xids.data()), xids); } public void search_preassigned(long n, SWIGTYPE_p_float x, long k, LongVector assign, SWIGTYPE_p_float centroid_dis, SWIGTYPE_p_float distances, LongVector labels, boolean store_pairs, IVFSearchParameters params, IndexIVFStats stats) { swigfaissJNI.IndexIVFFlatDedup_search_preassigned__SWIG_0(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_long_long.getCPtr(assign.data()), assign, SWIGTYPE_p_float.getCPtr(centroid_dis), SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, store_pairs, IVFSearchParameters.getCPtr(params), params, IndexIVFStats.getCPtr(stats), stats); } public void search_preassigned(long n, SWIGTYPE_p_float x, long k, LongVector assign, SWIGTYPE_p_float centroid_dis, SWIGTYPE_p_float distances, LongVector labels, boolean store_pairs, IVFSearchParameters params) { swigfaissJNI.IndexIVFFlatDedup_search_preassigned__SWIG_1(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_long_long.getCPtr(assign.data()), assign, SWIGTYPE_p_float.getCPtr(centroid_dis), SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, store_pairs, IVFSearchParameters.getCPtr(params), params); } public void search_preassigned(long n, SWIGTYPE_p_float x, long k, LongVector assign, SWIGTYPE_p_float centroid_dis, SWIGTYPE_p_float distances, LongVector labels, boolean store_pairs) { swigfaissJNI.IndexIVFFlatDedup_search_preassigned__SWIG_2(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_long_long.getCPtr(assign.data()), assign, SWIGTYPE_p_float.getCPtr(centroid_dis), SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, store_pairs); } public long remove_ids(IDSelector sel) { return swigfaissJNI.IndexIVFFlatDedup_remove_ids(swigCPtr, this, IDSelector.getCPtr(sel), sel); } public void range_search(long n, SWIGTYPE_p_float x, float radius, RangeSearchResult result) { swigfaissJNI.IndexIVFFlatDedup_range_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), radius, RangeSearchResult.getCPtr(result), result); } public void update_vectors(int nv, LongVector idx, SWIGTYPE_p_float v) { swigfaissJNI.IndexIVFFlatDedup_update_vectors(swigCPtr, this, nv, SWIGTYPE_p_long_long.getCPtr(idx.data()), idx, SWIGTYPE_p_float.getCPtr(v)); } public void reconstruct_from_offset(long list_no, long offset, SWIGTYPE_p_float recons) { swigfaissJNI.IndexIVFFlatDedup_reconstruct_from_offset(swigCPtr, this, list_no, offset, SWIGTYPE_p_float.getCPtr(recons)); } public IndexIVFFlatDedup() { this(swigfaissJNI.new_IndexIVFFlatDedup__SWIG_2(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexIVFPQ.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexIVFPQ extends IndexIVF { private transient long swigCPtr; protected IndexIVFPQ(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexIVFPQ_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexIVFPQ obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexIVFPQ(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setBy_residual(boolean value) { swigfaissJNI.IndexIVFPQ_by_residual_set(swigCPtr, this, value); } public boolean getBy_residual() { return swigfaissJNI.IndexIVFPQ_by_residual_get(swigCPtr, this); } public void setPq(ProductQuantizer value) { swigfaissJNI.IndexIVFPQ_pq_set(swigCPtr, this, ProductQuantizer.getCPtr(value), value); } public ProductQuantizer getPq() { long cPtr = swigfaissJNI.IndexIVFPQ_pq_get(swigCPtr, this); return (cPtr == 0) ? null : new ProductQuantizer(cPtr, false); } public void setDo_polysemous_training(boolean value) { swigfaissJNI.IndexIVFPQ_do_polysemous_training_set(swigCPtr, this, value); } public boolean getDo_polysemous_training() { return swigfaissJNI.IndexIVFPQ_do_polysemous_training_get(swigCPtr, this); } public void setPolysemous_training(PolysemousTraining value) { swigfaissJNI.IndexIVFPQ_polysemous_training_set(swigCPtr, this, PolysemousTraining.getCPtr(value), value); } public PolysemousTraining getPolysemous_training() { long cPtr = swigfaissJNI.IndexIVFPQ_polysemous_training_get(swigCPtr, this); return (cPtr == 0) ? null : new PolysemousTraining(cPtr, false); } public void setScan_table_threshold(long value) { swigfaissJNI.IndexIVFPQ_scan_table_threshold_set(swigCPtr, this, value); } public long getScan_table_threshold() { return swigfaissJNI.IndexIVFPQ_scan_table_threshold_get(swigCPtr, this); } public void setPolysemous_ht(int value) { swigfaissJNI.IndexIVFPQ_polysemous_ht_set(swigCPtr, this, value); } public int getPolysemous_ht() { return swigfaissJNI.IndexIVFPQ_polysemous_ht_get(swigCPtr, this); } public void setUse_precomputed_table(int value) { swigfaissJNI.IndexIVFPQ_use_precomputed_table_set(swigCPtr, this, value); } public int getUse_precomputed_table() { return swigfaissJNI.IndexIVFPQ_use_precomputed_table_get(swigCPtr, this); } public void setPrecomputed_table(SWIGTYPE_p_AlignedTableT_float_t value) { swigfaissJNI.IndexIVFPQ_precomputed_table_set(swigCPtr, this, SWIGTYPE_p_AlignedTableT_float_t.getCPtr(value)); } public SWIGTYPE_p_AlignedTableT_float_t getPrecomputed_table() { return new SWIGTYPE_p_AlignedTableT_float_t(swigfaissJNI.IndexIVFPQ_precomputed_table_get(swigCPtr, this), true); } public IndexIVFPQ(Index quantizer, long d, long nlist, long M, long nbits_per_idx, MetricType metric) { this(swigfaissJNI.new_IndexIVFPQ__SWIG_0(Index.getCPtr(quantizer), quantizer, d, nlist, M, nbits_per_idx, metric.swigValue()), true); } public IndexIVFPQ(Index quantizer, long d, long nlist, long M, long nbits_per_idx) { this(swigfaissJNI.new_IndexIVFPQ__SWIG_1(Index.getCPtr(quantizer), quantizer, d, nlist, M, nbits_per_idx), true); } public void encode_vectors(long n, SWIGTYPE_p_float x, LongVector list_nos, SWIGTYPE_p_unsigned_char codes, boolean include_listnos) { swigfaissJNI.IndexIVFPQ_encode_vectors__SWIG_0(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(list_nos.data()), list_nos, SWIGTYPE_p_unsigned_char.getCPtr(codes), include_listnos); } public void encode_vectors(long n, SWIGTYPE_p_float x, LongVector list_nos, SWIGTYPE_p_unsigned_char codes) { swigfaissJNI.IndexIVFPQ_encode_vectors__SWIG_1(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(list_nos.data()), list_nos, SWIGTYPE_p_unsigned_char.getCPtr(codes)); } public void sa_decode(long n, SWIGTYPE_p_unsigned_char bytes, SWIGTYPE_p_float x) { swigfaissJNI.IndexIVFPQ_sa_decode(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(bytes), SWIGTYPE_p_float.getCPtr(x)); } public void add_core(long n, SWIGTYPE_p_float x, LongVector xids, LongVector precomputed_idx) { swigfaissJNI.IndexIVFPQ_add_core(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(xids.data()), xids, SWIGTYPE_p_long_long.getCPtr(precomputed_idx.data()), precomputed_idx); } public void add_core_o(long n, SWIGTYPE_p_float x, LongVector xids, SWIGTYPE_p_float residuals_2, LongVector precomputed_idx) { swigfaissJNI.IndexIVFPQ_add_core_o__SWIG_0(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(xids.data()), xids, SWIGTYPE_p_float.getCPtr(residuals_2), SWIGTYPE_p_long_long.getCPtr(precomputed_idx.data()), precomputed_idx); } public void add_core_o(long n, SWIGTYPE_p_float x, LongVector xids, SWIGTYPE_p_float residuals_2) { swigfaissJNI.IndexIVFPQ_add_core_o__SWIG_1(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(xids.data()), xids, SWIGTYPE_p_float.getCPtr(residuals_2)); } public void train_residual(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexIVFPQ_train_residual(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void train_residual_o(long n, SWIGTYPE_p_float x, SWIGTYPE_p_float residuals_2) { swigfaissJNI.IndexIVFPQ_train_residual_o(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_float.getCPtr(residuals_2)); } public void reconstruct_from_offset(long list_no, long offset, SWIGTYPE_p_float recons) { swigfaissJNI.IndexIVFPQ_reconstruct_from_offset(swigCPtr, this, list_no, offset, SWIGTYPE_p_float.getCPtr(recons)); } public long find_duplicates(LongVector ids, SWIGTYPE_p_unsigned_long lims) { return swigfaissJNI.IndexIVFPQ_find_duplicates(swigCPtr, this, SWIGTYPE_p_long_long.getCPtr(ids.data()), ids, SWIGTYPE_p_unsigned_long.getCPtr(lims)); } public void encode(long key, SWIGTYPE_p_float x, SWIGTYPE_p_unsigned_char code) { swigfaissJNI.IndexIVFPQ_encode(swigCPtr, this, key, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_unsigned_char.getCPtr(code)); } public void encode_multiple(long n, LongVector keys, SWIGTYPE_p_float x, SWIGTYPE_p_unsigned_char codes, boolean compute_keys) { swigfaissJNI.IndexIVFPQ_encode_multiple__SWIG_0(swigCPtr, this, n, SWIGTYPE_p_long_long.getCPtr(keys.data()), keys, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_unsigned_char.getCPtr(codes), compute_keys); } public void encode_multiple(long n, LongVector keys, SWIGTYPE_p_float x, SWIGTYPE_p_unsigned_char codes) { swigfaissJNI.IndexIVFPQ_encode_multiple__SWIG_1(swigCPtr, this, n, SWIGTYPE_p_long_long.getCPtr(keys.data()), keys, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_unsigned_char.getCPtr(codes)); } public void decode_multiple(long n, LongVector keys, SWIGTYPE_p_unsigned_char xcodes, SWIGTYPE_p_float x) { swigfaissJNI.IndexIVFPQ_decode_multiple(swigCPtr, this, n, SWIGTYPE_p_long_long.getCPtr(keys.data()), keys, SWIGTYPE_p_unsigned_char.getCPtr(xcodes), SWIGTYPE_p_float.getCPtr(x)); } public SWIGTYPE_p_faiss__InvertedListScanner get_InvertedListScanner(boolean store_pairs) { long cPtr = swigfaissJNI.IndexIVFPQ_get_InvertedListScanner(swigCPtr, this, store_pairs); return (cPtr == 0) ? null : new SWIGTYPE_p_faiss__InvertedListScanner(cPtr, false); } public void precompute_table() { swigfaissJNI.IndexIVFPQ_precompute_table(swigCPtr, this); } public IndexIVFPQ() { this(swigfaissJNI.new_IndexIVFPQ__SWIG_2(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexIVFPQStats.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexIVFPQStats { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected IndexIVFPQStats(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(IndexIVFPQStats obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexIVFPQStats(swigCPtr); } swigCPtr = 0; } } public void setNrefine(long value) { swigfaissJNI.IndexIVFPQStats_nrefine_set(swigCPtr, this, value); } public long getNrefine() { return swigfaissJNI.IndexIVFPQStats_nrefine_get(swigCPtr, this); } public void setN_hamming_pass(long value) { swigfaissJNI.IndexIVFPQStats_n_hamming_pass_set(swigCPtr, this, value); } public long getN_hamming_pass() { return swigfaissJNI.IndexIVFPQStats_n_hamming_pass_get(swigCPtr, this); } public void setSearch_cycles(long value) { swigfaissJNI.IndexIVFPQStats_search_cycles_set(swigCPtr, this, value); } public long getSearch_cycles() { return swigfaissJNI.IndexIVFPQStats_search_cycles_get(swigCPtr, this); } public void setRefine_cycles(long value) { swigfaissJNI.IndexIVFPQStats_refine_cycles_set(swigCPtr, this, value); } public long getRefine_cycles() { return swigfaissJNI.IndexIVFPQStats_refine_cycles_get(swigCPtr, this); } public IndexIVFPQStats() { this(swigfaissJNI.new_IndexIVFPQStats(), true); } public void reset() { swigfaissJNI.IndexIVFPQStats_reset(swigCPtr, this); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexIVFScalarQuantizer.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexIVFScalarQuantizer extends IndexIVF { private transient long swigCPtr; protected IndexIVFScalarQuantizer(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexIVFScalarQuantizer_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexIVFScalarQuantizer obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexIVFScalarQuantizer(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setSq(SWIGTYPE_p_ScalarQuantizer value) { swigfaissJNI.IndexIVFScalarQuantizer_sq_set(swigCPtr, this, SWIGTYPE_p_ScalarQuantizer.getCPtr(value)); } public SWIGTYPE_p_ScalarQuantizer getSq() { return new SWIGTYPE_p_ScalarQuantizer(swigfaissJNI.IndexIVFScalarQuantizer_sq_get(swigCPtr, this), true); } public void setBy_residual(boolean value) { swigfaissJNI.IndexIVFScalarQuantizer_by_residual_set(swigCPtr, this, value); } public boolean getBy_residual() { return swigfaissJNI.IndexIVFScalarQuantizer_by_residual_get(swigCPtr, this); } public IndexIVFScalarQuantizer(Index quantizer, long d, long nlist, SWIGTYPE_p_ScalarQuantizer__QuantizerType qtype, MetricType metric, boolean encode_residual) { this(swigfaissJNI.new_IndexIVFScalarQuantizer__SWIG_0(Index.getCPtr(quantizer), quantizer, d, nlist, SWIGTYPE_p_ScalarQuantizer__QuantizerType.getCPtr(qtype), metric.swigValue(), encode_residual), true); } public IndexIVFScalarQuantizer(Index quantizer, long d, long nlist, SWIGTYPE_p_ScalarQuantizer__QuantizerType qtype, MetricType metric) { this(swigfaissJNI.new_IndexIVFScalarQuantizer__SWIG_1(Index.getCPtr(quantizer), quantizer, d, nlist, SWIGTYPE_p_ScalarQuantizer__QuantizerType.getCPtr(qtype), metric.swigValue()), true); } public IndexIVFScalarQuantizer(Index quantizer, long d, long nlist, SWIGTYPE_p_ScalarQuantizer__QuantizerType qtype) { this(swigfaissJNI.new_IndexIVFScalarQuantizer__SWIG_2(Index.getCPtr(quantizer), quantizer, d, nlist, SWIGTYPE_p_ScalarQuantizer__QuantizerType.getCPtr(qtype)), true); } public IndexIVFScalarQuantizer() { this(swigfaissJNI.new_IndexIVFScalarQuantizer__SWIG_3(), true); } public void train_residual(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexIVFScalarQuantizer_train_residual(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void encode_vectors(long n, SWIGTYPE_p_float x, LongVector list_nos, SWIGTYPE_p_unsigned_char codes, boolean include_listnos) { swigfaissJNI.IndexIVFScalarQuantizer_encode_vectors__SWIG_0(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(list_nos.data()), list_nos, SWIGTYPE_p_unsigned_char.getCPtr(codes), include_listnos); } public void encode_vectors(long n, SWIGTYPE_p_float x, LongVector list_nos, SWIGTYPE_p_unsigned_char codes) { swigfaissJNI.IndexIVFScalarQuantizer_encode_vectors__SWIG_1(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(list_nos.data()), list_nos, SWIGTYPE_p_unsigned_char.getCPtr(codes)); } public void add_core(long n, SWIGTYPE_p_float x, LongVector xids, LongVector precomputed_idx) { swigfaissJNI.IndexIVFScalarQuantizer_add_core(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(xids.data()), xids, SWIGTYPE_p_long_long.getCPtr(precomputed_idx.data()), precomputed_idx); } public SWIGTYPE_p_faiss__InvertedListScanner get_InvertedListScanner(boolean store_pairs) { long cPtr = swigfaissJNI.IndexIVFScalarQuantizer_get_InvertedListScanner(swigCPtr, this, store_pairs); return (cPtr == 0) ? null : new SWIGTYPE_p_faiss__InvertedListScanner(cPtr, false); } public void reconstruct_from_offset(long list_no, long offset, SWIGTYPE_p_float recons) { swigfaissJNI.IndexIVFScalarQuantizer_reconstruct_from_offset(swigCPtr, this, list_no, offset, SWIGTYPE_p_float.getCPtr(recons)); } public void sa_decode(long n, SWIGTYPE_p_unsigned_char bytes, SWIGTYPE_p_float x) { swigfaissJNI.IndexIVFScalarQuantizer_sa_decode(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(bytes), SWIGTYPE_p_float.getCPtr(x)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexIVFStats.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexIVFStats { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected IndexIVFStats(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(IndexIVFStats obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexIVFStats(swigCPtr); } swigCPtr = 0; } } public void setNq(long value) { swigfaissJNI.IndexIVFStats_nq_set(swigCPtr, this, value); } public long getNq() { return swigfaissJNI.IndexIVFStats_nq_get(swigCPtr, this); } public void setNlist(long value) { swigfaissJNI.IndexIVFStats_nlist_set(swigCPtr, this, value); } public long getNlist() { return swigfaissJNI.IndexIVFStats_nlist_get(swigCPtr, this); } public void setNdis(long value) { swigfaissJNI.IndexIVFStats_ndis_set(swigCPtr, this, value); } public long getNdis() { return swigfaissJNI.IndexIVFStats_ndis_get(swigCPtr, this); } public void setNheap_updates(long value) { swigfaissJNI.IndexIVFStats_nheap_updates_set(swigCPtr, this, value); } public long getNheap_updates() { return swigfaissJNI.IndexIVFStats_nheap_updates_get(swigCPtr, this); } public void setQuantization_time(double value) { swigfaissJNI.IndexIVFStats_quantization_time_set(swigCPtr, this, value); } public double getQuantization_time() { return swigfaissJNI.IndexIVFStats_quantization_time_get(swigCPtr, this); } public void setSearch_time(double value) { swigfaissJNI.IndexIVFStats_search_time_set(swigCPtr, this, value); } public double getSearch_time() { return swigfaissJNI.IndexIVFStats_search_time_get(swigCPtr, this); } public IndexIVFStats() { this(swigfaissJNI.new_IndexIVFStats(), true); } public void reset() { swigfaissJNI.IndexIVFStats_reset(swigCPtr, this); } public void add(IndexIVFStats other) { swigfaissJNI.IndexIVFStats_add(swigCPtr, this, IndexIVFStats.getCPtr(other), other); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexLSH.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexLSH extends IndexFlatCodes { private transient long swigCPtr; protected IndexLSH(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexLSH_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexLSH obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexLSH(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setNbits(int value) { swigfaissJNI.IndexLSH_nbits_set(swigCPtr, this, value); } public int getNbits() { return swigfaissJNI.IndexLSH_nbits_get(swigCPtr, this); } public void setRotate_data(boolean value) { swigfaissJNI.IndexLSH_rotate_data_set(swigCPtr, this, value); } public boolean getRotate_data() { return swigfaissJNI.IndexLSH_rotate_data_get(swigCPtr, this); } public void setTrain_thresholds(boolean value) { swigfaissJNI.IndexLSH_train_thresholds_set(swigCPtr, this, value); } public boolean getTrain_thresholds() { return swigfaissJNI.IndexLSH_train_thresholds_get(swigCPtr, this); } public void setRrot(RandomRotationMatrix value) { swigfaissJNI.IndexLSH_rrot_set(swigCPtr, this, RandomRotationMatrix.getCPtr(value), value); } public RandomRotationMatrix getRrot() { long cPtr = swigfaissJNI.IndexLSH_rrot_get(swigCPtr, this); return (cPtr == 0) ? null : new RandomRotationMatrix(cPtr, false); } public void setThresholds(FloatVector value) { swigfaissJNI.IndexLSH_thresholds_set(swigCPtr, this, FloatVector.getCPtr(value), value); } public FloatVector getThresholds() { long cPtr = swigfaissJNI.IndexLSH_thresholds_get(swigCPtr, this); return (cPtr == 0) ? null : new FloatVector(cPtr, false); } public IndexLSH(long d, int nbits, boolean rotate_data, boolean train_thresholds) { this(swigfaissJNI.new_IndexLSH__SWIG_0(d, nbits, rotate_data, train_thresholds), true); } public IndexLSH(long d, int nbits, boolean rotate_data) { this(swigfaissJNI.new_IndexLSH__SWIG_1(d, nbits, rotate_data), true); } public IndexLSH(long d, int nbits) { this(swigfaissJNI.new_IndexLSH__SWIG_2(d, nbits), true); } public SWIGTYPE_p_float apply_preprocess(long n, SWIGTYPE_p_float x) { long cPtr = swigfaissJNI.IndexLSH_apply_preprocess(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexLSH_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.IndexLSH_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void transfer_thresholds(LinearTransform vt) { swigfaissJNI.IndexLSH_transfer_thresholds(swigCPtr, this, LinearTransform.getCPtr(vt), vt); } public IndexLSH() { this(swigfaissJNI.new_IndexLSH__SWIG_3(), true); } public void sa_encode(long n, SWIGTYPE_p_float x, SWIGTYPE_p_unsigned_char bytes) { swigfaissJNI.IndexLSH_sa_encode(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_unsigned_char.getCPtr(bytes)); } public void sa_decode(long n, SWIGTYPE_p_unsigned_char bytes, SWIGTYPE_p_float x) { swigfaissJNI.IndexLSH_sa_decode(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(bytes), SWIGTYPE_p_float.getCPtr(x)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexPQ.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexPQ extends IndexFlatCodes { private transient long swigCPtr; protected IndexPQ(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexPQ_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexPQ obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexPQ(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setPq(ProductQuantizer value) { swigfaissJNI.IndexPQ_pq_set(swigCPtr, this, ProductQuantizer.getCPtr(value), value); } public ProductQuantizer getPq() { long cPtr = swigfaissJNI.IndexPQ_pq_get(swigCPtr, this); return (cPtr == 0) ? null : new ProductQuantizer(cPtr, false); } public IndexPQ(int d, long M, long nbits, MetricType metric) { this(swigfaissJNI.new_IndexPQ__SWIG_0(d, M, nbits, metric.swigValue()), true); } public IndexPQ(int d, long M, long nbits) { this(swigfaissJNI.new_IndexPQ__SWIG_1(d, M, nbits), true); } public IndexPQ() { this(swigfaissJNI.new_IndexPQ__SWIG_2(), true); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexPQ_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.IndexPQ_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void sa_encode(long n, SWIGTYPE_p_float x, SWIGTYPE_p_unsigned_char bytes) { swigfaissJNI.IndexPQ_sa_encode(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_unsigned_char.getCPtr(bytes)); } public void sa_decode(long n, SWIGTYPE_p_unsigned_char bytes, SWIGTYPE_p_float x) { swigfaissJNI.IndexPQ_sa_decode(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(bytes), SWIGTYPE_p_float.getCPtr(x)); } public DistanceComputer get_distance_computer() { long cPtr = swigfaissJNI.IndexPQ_get_distance_computer(swigCPtr, this); return (cPtr == 0) ? null : new DistanceComputer(cPtr, false); } public void setDo_polysemous_training(boolean value) { swigfaissJNI.IndexPQ_do_polysemous_training_set(swigCPtr, this, value); } public boolean getDo_polysemous_training() { return swigfaissJNI.IndexPQ_do_polysemous_training_get(swigCPtr, this); } public void setPolysemous_training(PolysemousTraining value) { swigfaissJNI.IndexPQ_polysemous_training_set(swigCPtr, this, PolysemousTraining.getCPtr(value), value); } public PolysemousTraining getPolysemous_training() { long cPtr = swigfaissJNI.IndexPQ_polysemous_training_get(swigCPtr, this); return (cPtr == 0) ? null : new PolysemousTraining(cPtr, false); } public void setSearch_type(IndexPQ.Search_type_t value) { swigfaissJNI.IndexPQ_search_type_set(swigCPtr, this, value.swigValue()); } public IndexPQ.Search_type_t getSearch_type() { return IndexPQ.Search_type_t.swigToEnum(swigfaissJNI.IndexPQ_search_type_get(swigCPtr, this)); } public void setEncode_signs(boolean value) { swigfaissJNI.IndexPQ_encode_signs_set(swigCPtr, this, value); } public boolean getEncode_signs() { return swigfaissJNI.IndexPQ_encode_signs_get(swigCPtr, this); } public void setPolysemous_ht(int value) { swigfaissJNI.IndexPQ_polysemous_ht_set(swigCPtr, this, value); } public int getPolysemous_ht() { return swigfaissJNI.IndexPQ_polysemous_ht_get(swigCPtr, this); } public void search_core_polysemous(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.IndexPQ_search_core_polysemous(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void hamming_distance_histogram(long n, SWIGTYPE_p_float x, long nb, SWIGTYPE_p_float xb, LongVector dist_histogram) { swigfaissJNI.IndexPQ_hamming_distance_histogram(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), nb, SWIGTYPE_p_float.getCPtr(xb), SWIGTYPE_p_long_long.getCPtr(dist_histogram.data()), dist_histogram); } public void hamming_distance_table(long n, SWIGTYPE_p_float x, SWIGTYPE_p_int dis) { swigfaissJNI.IndexPQ_hamming_distance_table(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_int.getCPtr(dis)); } public final static class Search_type_t { public final static IndexPQ.Search_type_t ST_PQ = new IndexPQ.Search_type_t("ST_PQ"); public final static IndexPQ.Search_type_t ST_HE = new IndexPQ.Search_type_t("ST_HE"); public final static IndexPQ.Search_type_t ST_generalized_HE = new IndexPQ.Search_type_t("ST_generalized_HE"); public final static IndexPQ.Search_type_t ST_SDC = new IndexPQ.Search_type_t("ST_SDC"); public final static IndexPQ.Search_type_t ST_polysemous = new IndexPQ.Search_type_t("ST_polysemous"); public final static IndexPQ.Search_type_t ST_polysemous_generalize = new IndexPQ.Search_type_t("ST_polysemous_generalize"); public final int swigValue() { return swigValue; } public String toString() { return swigName; } public static Search_type_t swigToEnum(int swigValue) { if (swigValue < swigValues.length && swigValue >= 0 && swigValues[swigValue].swigValue == swigValue) return swigValues[swigValue]; for (int i = 0; i < swigValues.length; i++) if (swigValues[i].swigValue == swigValue) return swigValues[i]; throw new IllegalArgumentException("No enum " + Search_type_t.class + " with value " + swigValue); } private Search_type_t(String swigName) { this.swigName = swigName; this.swigValue = swigNext++; } private Search_type_t(String swigName, int swigValue) { this.swigName = swigName; this.swigValue = swigValue; swigNext = swigValue+1; } private Search_type_t(String swigName, Search_type_t swigEnum) { this.swigName = swigName; this.swigValue = swigEnum.swigValue; swigNext = this.swigValue+1; } private static Search_type_t[] swigValues = { ST_PQ, ST_HE, ST_generalized_HE, ST_SDC, ST_polysemous, ST_polysemous_generalize }; private static int swigNext = 0; private final int swigValue; private final String swigName; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexPQStats.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexPQStats { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected IndexPQStats(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(IndexPQStats obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexPQStats(swigCPtr); } swigCPtr = 0; } } public void setNq(long value) { swigfaissJNI.IndexPQStats_nq_set(swigCPtr, this, value); } public long getNq() { return swigfaissJNI.IndexPQStats_nq_get(swigCPtr, this); } public void setNcode(long value) { swigfaissJNI.IndexPQStats_ncode_set(swigCPtr, this, value); } public long getNcode() { return swigfaissJNI.IndexPQStats_ncode_get(swigCPtr, this); } public void setN_hamming_pass(long value) { swigfaissJNI.IndexPQStats_n_hamming_pass_set(swigCPtr, this, value); } public long getN_hamming_pass() { return swigfaissJNI.IndexPQStats_n_hamming_pass_get(swigCPtr, this); } public IndexPQStats() { this(swigfaissJNI.new_IndexPQStats(), true); } public void reset() { swigfaissJNI.IndexPQStats_reset(swigCPtr, this); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexRefine.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexRefine extends Index { private transient long swigCPtr; protected IndexRefine(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexRefine_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexRefine obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexRefine(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setBase_index(Index value) { swigfaissJNI.IndexRefine_base_index_set(swigCPtr, this, Index.getCPtr(value), value); } public Index getBase_index() { long cPtr = swigfaissJNI.IndexRefine_base_index_get(swigCPtr, this); return (cPtr == 0) ? null : new Index(cPtr, false); } public void setRefine_index(Index value) { swigfaissJNI.IndexRefine_refine_index_set(swigCPtr, this, Index.getCPtr(value), value); } public Index getRefine_index() { long cPtr = swigfaissJNI.IndexRefine_refine_index_get(swigCPtr, this); return (cPtr == 0) ? null : new Index(cPtr, false); } public void setOwn_fields(boolean value) { swigfaissJNI.IndexRefine_own_fields_set(swigCPtr, this, value); } public boolean getOwn_fields() { return swigfaissJNI.IndexRefine_own_fields_get(swigCPtr, this); } public void setOwn_refine_index(boolean value) { swigfaissJNI.IndexRefine_own_refine_index_set(swigCPtr, this, value); } public boolean getOwn_refine_index() { return swigfaissJNI.IndexRefine_own_refine_index_get(swigCPtr, this); } public void setK_factor(float value) { swigfaissJNI.IndexRefine_k_factor_set(swigCPtr, this, value); } public float getK_factor() { return swigfaissJNI.IndexRefine_k_factor_get(swigCPtr, this); } public IndexRefine(Index base_index, Index refine_index) { this(swigfaissJNI.new_IndexRefine__SWIG_0(Index.getCPtr(base_index), base_index, Index.getCPtr(refine_index), refine_index), true); } public IndexRefine() { this(swigfaissJNI.new_IndexRefine__SWIG_1(), true); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexRefine_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void add(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexRefine_add(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void reset() { swigfaissJNI.IndexRefine_reset(swigCPtr, this); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.IndexRefine_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void reconstruct(long key, SWIGTYPE_p_float recons) { swigfaissJNI.IndexRefine_reconstruct(swigCPtr, this, key, SWIGTYPE_p_float.getCPtr(recons)); } public long sa_code_size() { return swigfaissJNI.IndexRefine_sa_code_size(swigCPtr, this); } public void sa_encode(long n, SWIGTYPE_p_float x, SWIGTYPE_p_unsigned_char bytes) { swigfaissJNI.IndexRefine_sa_encode(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_unsigned_char.getCPtr(bytes)); } public void sa_decode(long n, SWIGTYPE_p_unsigned_char bytes, SWIGTYPE_p_float x) { swigfaissJNI.IndexRefine_sa_decode(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(bytes), SWIGTYPE_p_float.getCPtr(x)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexRefineFlat.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexRefineFlat extends IndexRefine { private transient long swigCPtr; protected IndexRefineFlat(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexRefineFlat_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexRefineFlat obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexRefineFlat(swigCPtr); } swigCPtr = 0; } super.delete(); } public IndexRefineFlat(Index base_index) { this(swigfaissJNI.new_IndexRefineFlat__SWIG_0(Index.getCPtr(base_index), base_index), true); } public IndexRefineFlat(Index base_index, SWIGTYPE_p_float xb) { this(swigfaissJNI.new_IndexRefineFlat__SWIG_1(Index.getCPtr(base_index), base_index, SWIGTYPE_p_float.getCPtr(xb)), true); } public IndexRefineFlat() { this(swigfaissJNI.new_IndexRefineFlat__SWIG_2(), true); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.IndexRefineFlat_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexScalarQuantizer.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexScalarQuantizer extends IndexFlatCodes { private transient long swigCPtr; protected IndexScalarQuantizer(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexScalarQuantizer_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexScalarQuantizer obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexScalarQuantizer(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setSq(SWIGTYPE_p_ScalarQuantizer value) { swigfaissJNI.IndexScalarQuantizer_sq_set(swigCPtr, this, SWIGTYPE_p_ScalarQuantizer.getCPtr(value)); } public SWIGTYPE_p_ScalarQuantizer getSq() { return new SWIGTYPE_p_ScalarQuantizer(swigfaissJNI.IndexScalarQuantizer_sq_get(swigCPtr, this), true); } public IndexScalarQuantizer(int d, SWIGTYPE_p_ScalarQuantizer__QuantizerType qtype, MetricType metric) { this(swigfaissJNI.new_IndexScalarQuantizer__SWIG_0(d, SWIGTYPE_p_ScalarQuantizer__QuantizerType.getCPtr(qtype), metric.swigValue()), true); } public IndexScalarQuantizer(int d, SWIGTYPE_p_ScalarQuantizer__QuantizerType qtype) { this(swigfaissJNI.new_IndexScalarQuantizer__SWIG_1(d, SWIGTYPE_p_ScalarQuantizer__QuantizerType.getCPtr(qtype)), true); } public IndexScalarQuantizer() { this(swigfaissJNI.new_IndexScalarQuantizer__SWIG_2(), true); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexScalarQuantizer_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.IndexScalarQuantizer_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public DistanceComputer get_distance_computer() { long cPtr = swigfaissJNI.IndexScalarQuantizer_get_distance_computer(swigCPtr, this); return (cPtr == 0) ? null : new DistanceComputer(cPtr, false); } public void sa_encode(long n, SWIGTYPE_p_float x, SWIGTYPE_p_unsigned_char bytes) { swigfaissJNI.IndexScalarQuantizer_sa_encode(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_unsigned_char.getCPtr(bytes)); } public void sa_decode(long n, SWIGTYPE_p_unsigned_char bytes, SWIGTYPE_p_float x) { swigfaissJNI.IndexScalarQuantizer_sa_decode(swigCPtr, this, n, SWIGTYPE_p_unsigned_char.getCPtr(bytes), SWIGTYPE_p_float.getCPtr(x)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexShards.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexShards { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected IndexShards(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(IndexShards obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexShards(swigCPtr); } swigCPtr = 0; } } public IndexShards(boolean threaded, boolean successive_ids) { this(swigfaissJNI.new_IndexShards__SWIG_0(threaded, successive_ids), true); } public IndexShards(boolean threaded) { this(swigfaissJNI.new_IndexShards__SWIG_1(threaded), true); } public IndexShards() { this(swigfaissJNI.new_IndexShards__SWIG_2(), true); } public IndexShards(int d, boolean threaded, boolean successive_ids) { this(swigfaissJNI.new_IndexShards__SWIG_3(d, threaded, successive_ids), true); } public IndexShards(int d, boolean threaded) { this(swigfaissJNI.new_IndexShards__SWIG_4(d, threaded), true); } public IndexShards(int d) { this(swigfaissJNI.new_IndexShards__SWIG_5(d), true); } public void add_shard(Index index) { swigfaissJNI.IndexShards_add_shard(swigCPtr, this, Index.getCPtr(index), index); } public void remove_shard(Index index) { swigfaissJNI.IndexShards_remove_shard(swigCPtr, this, Index.getCPtr(index), index); } public void add(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexShards_add(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void add_with_ids(long n, SWIGTYPE_p_float x, LongVector xids) { swigfaissJNI.IndexShards_add_with_ids(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_long_long.getCPtr(xids.data()), xids); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.IndexShards_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexShards_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void setSuccessive_ids(boolean value) { swigfaissJNI.IndexShards_successive_ids_set(swigCPtr, this, value); } public boolean getSuccessive_ids() { return swigfaissJNI.IndexShards_successive_ids_get(swigCPtr, this); } public void syncWithSubIndexes() { swigfaissJNI.IndexShards_syncWithSubIndexes(swigCPtr, this); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IndexSplitVectors.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IndexSplitVectors extends Index { private transient long swigCPtr; protected IndexSplitVectors(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IndexSplitVectors_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IndexSplitVectors obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IndexSplitVectors(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setOwn_fields(boolean value) { swigfaissJNI.IndexSplitVectors_own_fields_set(swigCPtr, this, value); } public boolean getOwn_fields() { return swigfaissJNI.IndexSplitVectors_own_fields_get(swigCPtr, this); } public void setThreaded(boolean value) { swigfaissJNI.IndexSplitVectors_threaded_set(swigCPtr, this, value); } public boolean getThreaded() { return swigfaissJNI.IndexSplitVectors_threaded_get(swigCPtr, this); } public void setSub_indexes(SWIGTYPE_p_std__vectorT_faiss__Index_p_t value) { swigfaissJNI.IndexSplitVectors_sub_indexes_set(swigCPtr, this, SWIGTYPE_p_std__vectorT_faiss__Index_p_t.getCPtr(value)); } public SWIGTYPE_p_std__vectorT_faiss__Index_p_t getSub_indexes() { long cPtr = swigfaissJNI.IndexSplitVectors_sub_indexes_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_faiss__Index_p_t(cPtr, false); } public void setSum_d(long value) { swigfaissJNI.IndexSplitVectors_sum_d_set(swigCPtr, this, value); } public long getSum_d() { return swigfaissJNI.IndexSplitVectors_sum_d_get(swigCPtr, this); } public void add_sub_index(Index arg0) { swigfaissJNI.IndexSplitVectors_add_sub_index(swigCPtr, this, Index.getCPtr(arg0), arg0); } public void sync_with_sub_indexes() { swigfaissJNI.IndexSplitVectors_sync_with_sub_indexes(swigCPtr, this); } public void add(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexSplitVectors_add(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.IndexSplitVectors_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.IndexSplitVectors_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void reset() { swigfaissJNI.IndexSplitVectors_reset(swigCPtr, this); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IntVector.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IntVector { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected IntVector(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(IntVector obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IntVector(swigCPtr); } swigCPtr = 0; } } public IntVector() { this(swigfaissJNI.new_IntVector(), true); } public void push_back(int arg0) { swigfaissJNI.IntVector_push_back(swigCPtr, this, arg0); } public void clear() { swigfaissJNI.IntVector_clear(swigCPtr, this); } public SWIGTYPE_p_int data() { long cPtr = swigfaissJNI.IntVector_data(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_int(cPtr, false); } public long size() { return swigfaissJNI.IntVector_size(swigCPtr, this); } public int at(long n) { return swigfaissJNI.IntVector_at(swigCPtr, this, n); } public void resize(long n) { swigfaissJNI.IntVector_resize(swigCPtr, this, n); } public void reserve(long n) { swigfaissJNI.IntVector_reserve(swigCPtr, this, n); } public void swap(IntVector other) { swigfaissJNI.IntVector_swap(swigCPtr, this, IntVector.getCPtr(other), other); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/InterruptCallback.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class InterruptCallback { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected InterruptCallback(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(InterruptCallback obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_InterruptCallback(swigCPtr); } swigCPtr = 0; } } public boolean want_interrupt() { return swigfaissJNI.InterruptCallback_want_interrupt(swigCPtr, this); } public static void clear_instance() { swigfaissJNI.InterruptCallback_clear_instance(); } public static void check() { swigfaissJNI.InterruptCallback_check(); } public static boolean is_interrupted() { return swigfaissJNI.InterruptCallback_is_interrupted(); } public static long get_period_hint(long flops) { return swigfaissJNI.InterruptCallback_get_period_hint(flops); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/IntersectionCriterion.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class IntersectionCriterion extends AutoTuneCriterion { private transient long swigCPtr; protected IntersectionCriterion(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.IntersectionCriterion_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(IntersectionCriterion obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_IntersectionCriterion(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setR(long value) { swigfaissJNI.IntersectionCriterion_R_set(swigCPtr, this, value); } public long getR() { return swigfaissJNI.IntersectionCriterion_R_get(swigCPtr, this); } public IntersectionCriterion(long nq, long R) { this(swigfaissJNI.new_IntersectionCriterion(nq, R), true); } public double evaluate(SWIGTYPE_p_float D, LongVector I) { return swigfaissJNI.IntersectionCriterion_evaluate(swigCPtr, this, SWIGTYPE_p_float.getCPtr(D), SWIGTYPE_p_long_long.getCPtr(I.data()), I); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/InvertedLists.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class InvertedLists { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected InvertedLists(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(InvertedLists obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_InvertedLists(swigCPtr); } swigCPtr = 0; } } public void setNlist(long value) { swigfaissJNI.InvertedLists_nlist_set(swigCPtr, this, value); } public long getNlist() { return swigfaissJNI.InvertedLists_nlist_get(swigCPtr, this); } public void setCode_size(long value) { swigfaissJNI.InvertedLists_code_size_set(swigCPtr, this, value); } public long getCode_size() { return swigfaissJNI.InvertedLists_code_size_get(swigCPtr, this); } public long list_size(long list_no) { return swigfaissJNI.InvertedLists_list_size(swigCPtr, this, list_no); } public SWIGTYPE_p_unsigned_char get_codes(long list_no) { long cPtr = swigfaissJNI.InvertedLists_get_codes(swigCPtr, this, list_no); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public LongVector get_ids(long list_no) { return new LongVector(swigfaissJNI.InvertedLists_get_ids(swigCPtr, this, list_no), false); } public void release_codes(long list_no, SWIGTYPE_p_unsigned_char codes) { swigfaissJNI.InvertedLists_release_codes(swigCPtr, this, list_no, SWIGTYPE_p_unsigned_char.getCPtr(codes)); } public void release_ids(long list_no, LongVector ids) { swigfaissJNI.InvertedLists_release_ids(swigCPtr, this, list_no, SWIGTYPE_p_long_long.getCPtr(ids.data()), ids); } public long get_single_id(long list_no, long offset) { return swigfaissJNI.InvertedLists_get_single_id(swigCPtr, this, list_no, offset); } public SWIGTYPE_p_unsigned_char get_single_code(long list_no, long offset) { long cPtr = swigfaissJNI.InvertedLists_get_single_code(swigCPtr, this, list_no, offset); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public void prefetch_lists(LongVector list_nos, int nlist) { swigfaissJNI.InvertedLists_prefetch_lists(swigCPtr, this, SWIGTYPE_p_long_long.getCPtr(list_nos.data()), list_nos, nlist); } public long add_entry(long list_no, long theid, SWIGTYPE_p_unsigned_char code) { return swigfaissJNI.InvertedLists_add_entry(swigCPtr, this, list_no, theid, SWIGTYPE_p_unsigned_char.getCPtr(code)); } public long add_entries(long list_no, long n_entry, LongVector ids, SWIGTYPE_p_unsigned_char code) { return swigfaissJNI.InvertedLists_add_entries(swigCPtr, this, list_no, n_entry, SWIGTYPE_p_long_long.getCPtr(ids.data()), ids, SWIGTYPE_p_unsigned_char.getCPtr(code)); } public void update_entry(long list_no, long offset, long id, SWIGTYPE_p_unsigned_char code) { swigfaissJNI.InvertedLists_update_entry(swigCPtr, this, list_no, offset, id, SWIGTYPE_p_unsigned_char.getCPtr(code)); } public void update_entries(long list_no, long offset, long n_entry, LongVector ids, SWIGTYPE_p_unsigned_char code) { swigfaissJNI.InvertedLists_update_entries(swigCPtr, this, list_no, offset, n_entry, SWIGTYPE_p_long_long.getCPtr(ids.data()), ids, SWIGTYPE_p_unsigned_char.getCPtr(code)); } public void resize(long list_no, long new_size) { swigfaissJNI.InvertedLists_resize(swigCPtr, this, list_no, new_size); } public void reset() { swigfaissJNI.InvertedLists_reset(swigCPtr, this); } public void merge_from(InvertedLists oivf, long add_id) { swigfaissJNI.InvertedLists_merge_from(swigCPtr, this, InvertedLists.getCPtr(oivf), oivf, add_id); } public double imbalance_factor() { return swigfaissJNI.InvertedLists_imbalance_factor(swigCPtr, this); } public void print_stats() { swigfaissJNI.InvertedLists_print_stats(swigCPtr, this); } public long compute_ntotal() { return swigfaissJNI.InvertedLists_compute_ntotal(swigCPtr, this); } static public class ScopedIds { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected ScopedIds(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(ScopedIds obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_InvertedLists_ScopedIds(swigCPtr); } swigCPtr = 0; } } public void setIl(InvertedLists value) { swigfaissJNI.InvertedLists_ScopedIds_il_set(swigCPtr, this, InvertedLists.getCPtr(value), value); } public InvertedLists getIl() { long cPtr = swigfaissJNI.InvertedLists_ScopedIds_il_get(swigCPtr, this); return (cPtr == 0) ? null : new InvertedLists(cPtr, false); } public void setIds(LongVector value) { swigfaissJNI.InvertedLists_ScopedIds_ids_set(swigCPtr, this, SWIGTYPE_p_long_long.getCPtr(value.data()), value); } public LongVector getIds() { return new LongVector(swigfaissJNI.InvertedLists_ScopedIds_ids_get(swigCPtr, this), false); } public void setList_no(long value) { swigfaissJNI.InvertedLists_ScopedIds_list_no_set(swigCPtr, this, value); } public long getList_no() { return swigfaissJNI.InvertedLists_ScopedIds_list_no_get(swigCPtr, this); } public ScopedIds(InvertedLists il, long list_no) { this(swigfaissJNI.new_InvertedLists_ScopedIds(InvertedLists.getCPtr(il), il, list_no), true); } public LongVector get() { return new LongVector(swigfaissJNI.InvertedLists_ScopedIds_get(swigCPtr, this), false); } } static public class ScopedCodes { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected ScopedCodes(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(ScopedCodes obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_InvertedLists_ScopedCodes(swigCPtr); } swigCPtr = 0; } } public void setIl(InvertedLists value) { swigfaissJNI.InvertedLists_ScopedCodes_il_set(swigCPtr, this, InvertedLists.getCPtr(value), value); } public InvertedLists getIl() { long cPtr = swigfaissJNI.InvertedLists_ScopedCodes_il_get(swigCPtr, this); return (cPtr == 0) ? null : new InvertedLists(cPtr, false); } public void setCodes(SWIGTYPE_p_unsigned_char value) { swigfaissJNI.InvertedLists_ScopedCodes_codes_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(value)); } public SWIGTYPE_p_unsigned_char getCodes() { long cPtr = swigfaissJNI.InvertedLists_ScopedCodes_codes_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public void setList_no(long value) { swigfaissJNI.InvertedLists_ScopedCodes_list_no_set(swigCPtr, this, value); } public long getList_no() { return swigfaissJNI.InvertedLists_ScopedCodes_list_no_get(swigCPtr, this); } public ScopedCodes(InvertedLists il, long list_no) { this(swigfaissJNI.new_InvertedLists_ScopedCodes__SWIG_0(InvertedLists.getCPtr(il), il, list_no), true); } public ScopedCodes(InvertedLists il, long list_no, long offset) { this(swigfaissJNI.new_InvertedLists_ScopedCodes__SWIG_1(InvertedLists.getCPtr(il), il, list_no, offset), true); } public SWIGTYPE_p_unsigned_char get() { long cPtr = swigfaissJNI.InvertedLists_ScopedCodes_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } } public final static long INVALID_CODE_SIZE = swigfaissJNI.InvertedLists_INVALID_CODE_SIZE_get(); } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/InvertedListsPtrVector.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class InvertedListsPtrVector { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected InvertedListsPtrVector(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(InvertedListsPtrVector obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_InvertedListsPtrVector(swigCPtr); } swigCPtr = 0; } } public InvertedListsPtrVector() { this(swigfaissJNI.new_InvertedListsPtrVector(), true); } public void push_back(InvertedLists arg0) { swigfaissJNI.InvertedListsPtrVector_push_back(swigCPtr, this, InvertedLists.getCPtr(arg0), arg0); } public void clear() { swigfaissJNI.InvertedListsPtrVector_clear(swigCPtr, this); } public SWIGTYPE_p_p_faiss__InvertedLists data() { long cPtr = swigfaissJNI.InvertedListsPtrVector_data(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_p_faiss__InvertedLists(cPtr, false); } public long size() { return swigfaissJNI.InvertedListsPtrVector_size(swigCPtr, this); } public InvertedLists at(long n) { long cPtr = swigfaissJNI.InvertedListsPtrVector_at(swigCPtr, this, n); return (cPtr == 0) ? null : new InvertedLists(cPtr, false); } public void resize(long n) { swigfaissJNI.InvertedListsPtrVector_resize(swigCPtr, this, n); } public void reserve(long n) { swigfaissJNI.InvertedListsPtrVector_reserve(swigCPtr, this, n); } public void swap(InvertedListsPtrVector other) { swigfaissJNI.InvertedListsPtrVector_swap(swigCPtr, this, InvertedListsPtrVector.getCPtr(other), other); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/Level1Quantizer.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class Level1Quantizer { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected Level1Quantizer(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(Level1Quantizer obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_Level1Quantizer(swigCPtr); } swigCPtr = 0; } } public void setQuantizer(Index value) { swigfaissJNI.Level1Quantizer_quantizer_set(swigCPtr, this, Index.getCPtr(value), value); } public Index getQuantizer() { long cPtr = swigfaissJNI.Level1Quantizer_quantizer_get(swigCPtr, this); return (cPtr == 0) ? null : new Index(cPtr, false); } public void setNlist(long value) { swigfaissJNI.Level1Quantizer_nlist_set(swigCPtr, this, value); } public long getNlist() { return swigfaissJNI.Level1Quantizer_nlist_get(swigCPtr, this); } public void setQuantizer_trains_alone(char value) { swigfaissJNI.Level1Quantizer_quantizer_trains_alone_set(swigCPtr, this, value); } public char getQuantizer_trains_alone() { return swigfaissJNI.Level1Quantizer_quantizer_trains_alone_get(swigCPtr, this); } public void setOwn_fields(boolean value) { swigfaissJNI.Level1Quantizer_own_fields_set(swigCPtr, this, value); } public boolean getOwn_fields() { return swigfaissJNI.Level1Quantizer_own_fields_get(swigCPtr, this); } public void setCp(ClusteringParameters value) { swigfaissJNI.Level1Quantizer_cp_set(swigCPtr, this, ClusteringParameters.getCPtr(value), value); } public ClusteringParameters getCp() { long cPtr = swigfaissJNI.Level1Quantizer_cp_get(swigCPtr, this); return (cPtr == 0) ? null : new ClusteringParameters(cPtr, false); } public void setClustering_index(Index value) { swigfaissJNI.Level1Quantizer_clustering_index_set(swigCPtr, this, Index.getCPtr(value), value); } public Index getClustering_index() { long cPtr = swigfaissJNI.Level1Quantizer_clustering_index_get(swigCPtr, this); return (cPtr == 0) ? null : new Index(cPtr, false); } public void train_q1(long n, SWIGTYPE_p_float x, boolean verbose, MetricType metric_type) { swigfaissJNI.Level1Quantizer_train_q1(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), verbose, metric_type.swigValue()); } public long coarse_code_size() { return swigfaissJNI.Level1Quantizer_coarse_code_size(swigCPtr, this); } public void encode_listno(long list_no, SWIGTYPE_p_unsigned_char code) { swigfaissJNI.Level1Quantizer_encode_listno(swigCPtr, this, list_no, SWIGTYPE_p_unsigned_char.getCPtr(code)); } public long decode_listno(SWIGTYPE_p_unsigned_char code) { return swigfaissJNI.Level1Quantizer_decode_listno(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(code)); } public Level1Quantizer(Index quantizer, long nlist) { this(swigfaissJNI.new_Level1Quantizer__SWIG_0(Index.getCPtr(quantizer), quantizer, nlist), true); } public Level1Quantizer() { this(swigfaissJNI.new_Level1Quantizer__SWIG_1(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/LinearTransform.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class LinearTransform extends VectorTransform { private transient long swigCPtr; protected LinearTransform(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.LinearTransform_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(LinearTransform obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_LinearTransform(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setHave_bias(boolean value) { swigfaissJNI.LinearTransform_have_bias_set(swigCPtr, this, value); } public boolean getHave_bias() { return swigfaissJNI.LinearTransform_have_bias_get(swigCPtr, this); } public void setIs_orthonormal(boolean value) { swigfaissJNI.LinearTransform_is_orthonormal_set(swigCPtr, this, value); } public boolean getIs_orthonormal() { return swigfaissJNI.LinearTransform_is_orthonormal_get(swigCPtr, this); } public void setA(FloatVector value) { swigfaissJNI.LinearTransform_A_set(swigCPtr, this, FloatVector.getCPtr(value), value); } public FloatVector getA() { long cPtr = swigfaissJNI.LinearTransform_A_get(swigCPtr, this); return (cPtr == 0) ? null : new FloatVector(cPtr, false); } public void setB(FloatVector value) { swigfaissJNI.LinearTransform_b_set(swigCPtr, this, FloatVector.getCPtr(value), value); } public FloatVector getB() { long cPtr = swigfaissJNI.LinearTransform_b_get(swigCPtr, this); return (cPtr == 0) ? null : new FloatVector(cPtr, false); } public LinearTransform(int d_in, int d_out, boolean have_bias) { this(swigfaissJNI.new_LinearTransform__SWIG_0(d_in, d_out, have_bias), true); } public LinearTransform(int d_in, int d_out) { this(swigfaissJNI.new_LinearTransform__SWIG_1(d_in, d_out), true); } public LinearTransform(int d_in) { this(swigfaissJNI.new_LinearTransform__SWIG_2(d_in), true); } public LinearTransform() { this(swigfaissJNI.new_LinearTransform__SWIG_3(), true); } public void apply_noalloc(long n, SWIGTYPE_p_float x, SWIGTYPE_p_float xt) { swigfaissJNI.LinearTransform_apply_noalloc(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_float.getCPtr(xt)); } public void transform_transpose(long n, SWIGTYPE_p_float y, SWIGTYPE_p_float x) { swigfaissJNI.LinearTransform_transform_transpose(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(y), SWIGTYPE_p_float.getCPtr(x)); } public void reverse_transform(long n, SWIGTYPE_p_float xt, SWIGTYPE_p_float x) { swigfaissJNI.LinearTransform_reverse_transform(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(xt), SWIGTYPE_p_float.getCPtr(x)); } public void set_is_orthonormal() { swigfaissJNI.LinearTransform_set_is_orthonormal(swigCPtr, this); } public void setVerbose(boolean value) { swigfaissJNI.LinearTransform_verbose_set(swigCPtr, this, value); } public boolean getVerbose() { return swigfaissJNI.LinearTransform_verbose_get(swigCPtr, this); } public void print_if_verbose(String name, DoubleVector mat, int n, int d) { swigfaissJNI.LinearTransform_print_if_verbose(swigCPtr, this, name, DoubleVector.getCPtr(mat), mat, n, d); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/LongVector.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class LongVector { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected LongVector(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(LongVector obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_LongVector(swigCPtr); } swigCPtr = 0; } } public LongVector() { this(swigfaissJNI.new_LongVector(), true); } public void push_back(long arg0) { swigfaissJNI.LongVector_push_back(swigCPtr, this, arg0); } public void clear() { swigfaissJNI.LongVector_clear(swigCPtr, this); } public SWIGTYPE_p_long_long data() { long cPtr = swigfaissJNI.LongVector_data(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_long_long(cPtr, false); } public long size() { return swigfaissJNI.LongVector_size(swigCPtr, this); } public long at(long n) { return swigfaissJNI.LongVector_at(swigCPtr, this, n); } public void resize(long n) { swigfaissJNI.LongVector_resize(swigCPtr, this, n); } public void reserve(long n) { swigfaissJNI.LongVector_reserve(swigCPtr, this, n); } public void swap(LongVector other) { swigfaissJNI.LongVector_swap(swigCPtr, this, LongVector.getCPtr(other), other); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/LongVectorVector.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class LongVectorVector { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected LongVectorVector(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(LongVectorVector obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_LongVectorVector(swigCPtr); } swigCPtr = 0; } } public LongVectorVector() { this(swigfaissJNI.new_LongVectorVector(), true); } public void push_back(SWIGTYPE_p_std__vectorT_long_t arg0) { swigfaissJNI.LongVectorVector_push_back(swigCPtr, this, SWIGTYPE_p_std__vectorT_long_t.getCPtr(arg0)); } public void clear() { swigfaissJNI.LongVectorVector_clear(swigCPtr, this); } public SWIGTYPE_p_std__vectorT_long_t data() { long cPtr = swigfaissJNI.LongVectorVector_data(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_long_t(cPtr, false); } public long size() { return swigfaissJNI.LongVectorVector_size(swigCPtr, this); } public SWIGTYPE_p_std__vectorT_long_t at(long n) { return new SWIGTYPE_p_std__vectorT_long_t(swigfaissJNI.LongVectorVector_at(swigCPtr, this, n), true); } public void resize(long n) { swigfaissJNI.LongVectorVector_resize(swigCPtr, this, n); } public void reserve(long n) { swigfaissJNI.LongVectorVector_reserve(swigCPtr, this, n); } public void swap(LongVectorVector other) { swigfaissJNI.LongVectorVector_swap(swigCPtr, this, LongVectorVector.getCPtr(other), other); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/MapLong2Long.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class MapLong2Long { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected MapLong2Long(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(MapLong2Long obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_MapLong2Long(swigCPtr); } swigCPtr = 0; } } public void setMap(SWIGTYPE_p_std__unordered_mapT_long_long_t value) { swigfaissJNI.MapLong2Long_map_set(swigCPtr, this, SWIGTYPE_p_std__unordered_mapT_long_long_t.getCPtr(value)); } public SWIGTYPE_p_std__unordered_mapT_long_long_t getMap() { return new SWIGTYPE_p_std__unordered_mapT_long_long_t(swigfaissJNI.MapLong2Long_map_get(swigCPtr, this), true); } public void add(long n, SWIGTYPE_p_long keys, SWIGTYPE_p_long vals) { swigfaissJNI.MapLong2Long_add(swigCPtr, this, n, SWIGTYPE_p_long.getCPtr(keys), SWIGTYPE_p_long.getCPtr(vals)); } public int search(int key) { return swigfaissJNI.MapLong2Long_search(swigCPtr, this, key); } public void search_multiple(long n, SWIGTYPE_p_long keys, SWIGTYPE_p_long vals) { swigfaissJNI.MapLong2Long_search_multiple(swigCPtr, this, n, SWIGTYPE_p_long.getCPtr(keys), SWIGTYPE_p_long.getCPtr(vals)); } public MapLong2Long() { this(swigfaissJNI.new_MapLong2Long(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/MaskedInvertedLists.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class MaskedInvertedLists extends ReadOnlyInvertedLists { private transient long swigCPtr; protected MaskedInvertedLists(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.MaskedInvertedLists_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(MaskedInvertedLists obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_MaskedInvertedLists(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setIl0(InvertedLists value) { swigfaissJNI.MaskedInvertedLists_il0_set(swigCPtr, this, InvertedLists.getCPtr(value), value); } public InvertedLists getIl0() { long cPtr = swigfaissJNI.MaskedInvertedLists_il0_get(swigCPtr, this); return (cPtr == 0) ? null : new InvertedLists(cPtr, false); } public void setIl1(InvertedLists value) { swigfaissJNI.MaskedInvertedLists_il1_set(swigCPtr, this, InvertedLists.getCPtr(value), value); } public InvertedLists getIl1() { long cPtr = swigfaissJNI.MaskedInvertedLists_il1_get(swigCPtr, this); return (cPtr == 0) ? null : new InvertedLists(cPtr, false); } public MaskedInvertedLists(InvertedLists il0, InvertedLists il1) { this(swigfaissJNI.new_MaskedInvertedLists(InvertedLists.getCPtr(il0), il0, InvertedLists.getCPtr(il1), il1), true); } public long list_size(long list_no) { return swigfaissJNI.MaskedInvertedLists_list_size(swigCPtr, this, list_no); } public SWIGTYPE_p_unsigned_char get_codes(long list_no) { long cPtr = swigfaissJNI.MaskedInvertedLists_get_codes(swigCPtr, this, list_no); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public LongVector get_ids(long list_no) { return new LongVector(swigfaissJNI.MaskedInvertedLists_get_ids(swigCPtr, this, list_no), false); } public void release_codes(long list_no, SWIGTYPE_p_unsigned_char codes) { swigfaissJNI.MaskedInvertedLists_release_codes(swigCPtr, this, list_no, SWIGTYPE_p_unsigned_char.getCPtr(codes)); } public void release_ids(long list_no, LongVector ids) { swigfaissJNI.MaskedInvertedLists_release_ids(swigCPtr, this, list_no, SWIGTYPE_p_long_long.getCPtr(ids.data()), ids); } public long get_single_id(long list_no, long offset) { return swigfaissJNI.MaskedInvertedLists_get_single_id(swigCPtr, this, list_no, offset); } public SWIGTYPE_p_unsigned_char get_single_code(long list_no, long offset) { long cPtr = swigfaissJNI.MaskedInvertedLists_get_single_code(swigCPtr, this, list_no, offset); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public void prefetch_lists(LongVector list_nos, int nlist) { swigfaissJNI.MaskedInvertedLists_prefetch_lists(swigCPtr, this, SWIGTYPE_p_long_long.getCPtr(list_nos.data()), list_nos, nlist); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/MetricType.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public final class MetricType { public final static MetricType METRIC_INNER_PRODUCT = new MetricType("METRIC_INNER_PRODUCT", swigfaissJNI.METRIC_INNER_PRODUCT_get()); public final static MetricType METRIC_L2 = new MetricType("METRIC_L2", swigfaissJNI.METRIC_L2_get()); public final static MetricType METRIC_L1 = new MetricType("METRIC_L1"); public final static MetricType METRIC_Linf = new MetricType("METRIC_Linf"); public final static MetricType METRIC_Lp = new MetricType("METRIC_Lp"); public final static MetricType METRIC_Canberra = new MetricType("METRIC_Canberra", swigfaissJNI.METRIC_Canberra_get()); public final static MetricType METRIC_BrayCurtis = new MetricType("METRIC_BrayCurtis"); public final static MetricType METRIC_JensenShannon = new MetricType("METRIC_JensenShannon"); public final int swigValue() { return swigValue; } public String toString() { return swigName; } public static MetricType swigToEnum(int swigValue) { if (swigValue < swigValues.length && swigValue >= 0 && swigValues[swigValue].swigValue == swigValue) return swigValues[swigValue]; for (int i = 0; i < swigValues.length; i++) if (swigValues[i].swigValue == swigValue) return swigValues[i]; throw new IllegalArgumentException("No enum " + MetricType.class + " with value " + swigValue); } private MetricType(String swigName) { this.swigName = swigName; this.swigValue = swigNext++; } private MetricType(String swigName, int swigValue) { this.swigName = swigName; this.swigValue = swigValue; swigNext = swigValue+1; } private MetricType(String swigName, MetricType swigEnum) { this.swigName = swigName; this.swigValue = swigEnum.swigValue; swigNext = this.swigValue+1; } private static MetricType[] swigValues = { METRIC_INNER_PRODUCT, METRIC_L2, METRIC_L1, METRIC_Linf, METRIC_Lp, METRIC_Canberra, METRIC_BrayCurtis, METRIC_JensenShannon }; private static int swigNext = 0; private final int swigValue; private final String swigName; } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/MultiIndexQuantizer.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class MultiIndexQuantizer extends Index { private transient long swigCPtr; protected MultiIndexQuantizer(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.MultiIndexQuantizer_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(MultiIndexQuantizer obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_MultiIndexQuantizer(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setPq(ProductQuantizer value) { swigfaissJNI.MultiIndexQuantizer_pq_set(swigCPtr, this, ProductQuantizer.getCPtr(value), value); } public ProductQuantizer getPq() { long cPtr = swigfaissJNI.MultiIndexQuantizer_pq_get(swigCPtr, this); return (cPtr == 0) ? null : new ProductQuantizer(cPtr, false); } public MultiIndexQuantizer(int d, long M, long nbits) { this(swigfaissJNI.new_MultiIndexQuantizer__SWIG_0(d, M, nbits), true); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.MultiIndexQuantizer_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.MultiIndexQuantizer_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public void add(long n, SWIGTYPE_p_float x) { swigfaissJNI.MultiIndexQuantizer_add(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void reset() { swigfaissJNI.MultiIndexQuantizer_reset(swigCPtr, this); } public MultiIndexQuantizer() { this(swigfaissJNI.new_MultiIndexQuantizer__SWIG_1(), true); } public void reconstruct(long key, SWIGTYPE_p_float recons) { swigfaissJNI.MultiIndexQuantizer_reconstruct(swigCPtr, this, key, SWIGTYPE_p_float.getCPtr(recons)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/MultiIndexQuantizer2.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class MultiIndexQuantizer2 extends MultiIndexQuantizer { private transient long swigCPtr; protected MultiIndexQuantizer2(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.MultiIndexQuantizer2_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(MultiIndexQuantizer2 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_MultiIndexQuantizer2(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setAssign_indexes(SWIGTYPE_p_std__vectorT_faiss__Index_p_t value) { swigfaissJNI.MultiIndexQuantizer2_assign_indexes_set(swigCPtr, this, SWIGTYPE_p_std__vectorT_faiss__Index_p_t.getCPtr(value)); } public SWIGTYPE_p_std__vectorT_faiss__Index_p_t getAssign_indexes() { long cPtr = swigfaissJNI.MultiIndexQuantizer2_assign_indexes_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_faiss__Index_p_t(cPtr, false); } public void setOwn_fields(boolean value) { swigfaissJNI.MultiIndexQuantizer2_own_fields_set(swigCPtr, this, value); } public boolean getOwn_fields() { return swigfaissJNI.MultiIndexQuantizer2_own_fields_get(swigCPtr, this); } public MultiIndexQuantizer2(int d, long M, long nbits, SWIGTYPE_p_p_faiss__Index indexes) { this(swigfaissJNI.new_MultiIndexQuantizer2__SWIG_0(d, M, nbits, SWIGTYPE_p_p_faiss__Index.getCPtr(indexes)), true); } public MultiIndexQuantizer2(int d, long nbits, Index assign_index_0, Index assign_index_1) { this(swigfaissJNI.new_MultiIndexQuantizer2__SWIG_1(d, nbits, Index.getCPtr(assign_index_0), assign_index_0, Index.getCPtr(assign_index_1), assign_index_1), true); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.MultiIndexQuantizer2_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void search(long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels) { swigfaissJNI.MultiIndexQuantizer2_search(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/NormalizationTransform.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class NormalizationTransform extends VectorTransform { private transient long swigCPtr; protected NormalizationTransform(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.NormalizationTransform_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(NormalizationTransform obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_NormalizationTransform(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setNorm(float value) { swigfaissJNI.NormalizationTransform_norm_set(swigCPtr, this, value); } public float getNorm() { return swigfaissJNI.NormalizationTransform_norm_get(swigCPtr, this); } public NormalizationTransform(int d, float norm) { this(swigfaissJNI.new_NormalizationTransform__SWIG_0(d, norm), true); } public NormalizationTransform(int d) { this(swigfaissJNI.new_NormalizationTransform__SWIG_1(d), true); } public NormalizationTransform() { this(swigfaissJNI.new_NormalizationTransform__SWIG_2(), true); } public void apply_noalloc(long n, SWIGTYPE_p_float x, SWIGTYPE_p_float xt) { swigfaissJNI.NormalizationTransform_apply_noalloc(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_float.getCPtr(xt)); } public void reverse_transform(long n, SWIGTYPE_p_float xt, SWIGTYPE_p_float x) { swigfaissJNI.NormalizationTransform_reverse_transform(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(xt), SWIGTYPE_p_float.getCPtr(x)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/OPQMatrix.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class OPQMatrix extends LinearTransform { private transient long swigCPtr; protected OPQMatrix(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.OPQMatrix_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(OPQMatrix obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_OPQMatrix(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setM(int value) { swigfaissJNI.OPQMatrix_M_set(swigCPtr, this, value); } public int getM() { return swigfaissJNI.OPQMatrix_M_get(swigCPtr, this); } public void setNiter(int value) { swigfaissJNI.OPQMatrix_niter_set(swigCPtr, this, value); } public int getNiter() { return swigfaissJNI.OPQMatrix_niter_get(swigCPtr, this); } public void setNiter_pq(int value) { swigfaissJNI.OPQMatrix_niter_pq_set(swigCPtr, this, value); } public int getNiter_pq() { return swigfaissJNI.OPQMatrix_niter_pq_get(swigCPtr, this); } public void setNiter_pq_0(int value) { swigfaissJNI.OPQMatrix_niter_pq_0_set(swigCPtr, this, value); } public int getNiter_pq_0() { return swigfaissJNI.OPQMatrix_niter_pq_0_get(swigCPtr, this); } public void setMax_train_points(long value) { swigfaissJNI.OPQMatrix_max_train_points_set(swigCPtr, this, value); } public long getMax_train_points() { return swigfaissJNI.OPQMatrix_max_train_points_get(swigCPtr, this); } public void setVerbose(boolean value) { swigfaissJNI.OPQMatrix_verbose_set(swigCPtr, this, value); } public boolean getVerbose() { return swigfaissJNI.OPQMatrix_verbose_get(swigCPtr, this); } public void setPq(ProductQuantizer value) { swigfaissJNI.OPQMatrix_pq_set(swigCPtr, this, ProductQuantizer.getCPtr(value), value); } public ProductQuantizer getPq() { long cPtr = swigfaissJNI.OPQMatrix_pq_get(swigCPtr, this); return (cPtr == 0) ? null : new ProductQuantizer(cPtr, false); } public OPQMatrix(int d, int M, int d2) { this(swigfaissJNI.new_OPQMatrix__SWIG_0(d, M, d2), true); } public OPQMatrix(int d, int M) { this(swigfaissJNI.new_OPQMatrix__SWIG_1(d, M), true); } public OPQMatrix(int d) { this(swigfaissJNI.new_OPQMatrix__SWIG_2(d), true); } public OPQMatrix() { this(swigfaissJNI.new_OPQMatrix__SWIG_3(), true); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.OPQMatrix_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/OnDiskInvertedLists.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class OnDiskInvertedLists extends InvertedLists { private transient long swigCPtr; protected OnDiskInvertedLists(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.OnDiskInvertedLists_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(OnDiskInvertedLists obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_OnDiskInvertedLists(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setLists(SWIGTYPE_p_std__vectorT_faiss__OnDiskOneList_t value) { swigfaissJNI.OnDiskInvertedLists_lists_set(swigCPtr, this, SWIGTYPE_p_std__vectorT_faiss__OnDiskOneList_t.getCPtr(value)); } public SWIGTYPE_p_std__vectorT_faiss__OnDiskOneList_t getLists() { long cPtr = swigfaissJNI.OnDiskInvertedLists_lists_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_faiss__OnDiskOneList_t(cPtr, false); } static public class Slot { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected Slot(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(Slot obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_OnDiskInvertedLists_Slot(swigCPtr); } swigCPtr = 0; } } public void setOffset(long value) { swigfaissJNI.OnDiskInvertedLists_Slot_offset_set(swigCPtr, this, value); } public long getOffset() { return swigfaissJNI.OnDiskInvertedLists_Slot_offset_get(swigCPtr, this); } public void setCapacity(long value) { swigfaissJNI.OnDiskInvertedLists_Slot_capacity_set(swigCPtr, this, value); } public long getCapacity() { return swigfaissJNI.OnDiskInvertedLists_Slot_capacity_get(swigCPtr, this); } public Slot(long offset, long capacity) { this(swigfaissJNI.new_OnDiskInvertedLists_Slot__SWIG_0(offset, capacity), true); } public Slot() { this(swigfaissJNI.new_OnDiskInvertedLists_Slot__SWIG_1(), true); } } public void setSlots(SWIGTYPE_p_std__listT_faiss__OnDiskInvertedLists__Slot_t value) { swigfaissJNI.OnDiskInvertedLists_slots_set(swigCPtr, this, SWIGTYPE_p_std__listT_faiss__OnDiskInvertedLists__Slot_t.getCPtr(value)); } public SWIGTYPE_p_std__listT_faiss__OnDiskInvertedLists__Slot_t getSlots() { long cPtr = swigfaissJNI.OnDiskInvertedLists_slots_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__listT_faiss__OnDiskInvertedLists__Slot_t(cPtr, false); } public void setFilename(String value) { swigfaissJNI.OnDiskInvertedLists_filename_set(swigCPtr, this, value); } public String getFilename() { return swigfaissJNI.OnDiskInvertedLists_filename_get(swigCPtr, this); } public void setTotsize(long value) { swigfaissJNI.OnDiskInvertedLists_totsize_set(swigCPtr, this, value); } public long getTotsize() { return swigfaissJNI.OnDiskInvertedLists_totsize_get(swigCPtr, this); } public void setPtr(SWIGTYPE_p_unsigned_char value) { swigfaissJNI.OnDiskInvertedLists_ptr_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(value)); } public SWIGTYPE_p_unsigned_char getPtr() { long cPtr = swigfaissJNI.OnDiskInvertedLists_ptr_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public void setRead_only(boolean value) { swigfaissJNI.OnDiskInvertedLists_read_only_set(swigCPtr, this, value); } public boolean getRead_only() { return swigfaissJNI.OnDiskInvertedLists_read_only_get(swigCPtr, this); } public OnDiskInvertedLists(long nlist, long code_size, String filename) { this(swigfaissJNI.new_OnDiskInvertedLists__SWIG_0(nlist, code_size, filename), true); } public long list_size(long list_no) { return swigfaissJNI.OnDiskInvertedLists_list_size(swigCPtr, this, list_no); } public SWIGTYPE_p_unsigned_char get_codes(long list_no) { long cPtr = swigfaissJNI.OnDiskInvertedLists_get_codes(swigCPtr, this, list_no); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public LongVector get_ids(long list_no) { return new LongVector(swigfaissJNI.OnDiskInvertedLists_get_ids(swigCPtr, this, list_no), false); } public long add_entries(long list_no, long n_entry, LongVector ids, SWIGTYPE_p_unsigned_char code) { return swigfaissJNI.OnDiskInvertedLists_add_entries(swigCPtr, this, list_no, n_entry, SWIGTYPE_p_long_long.getCPtr(ids.data()), ids, SWIGTYPE_p_unsigned_char.getCPtr(code)); } public void update_entries(long list_no, long offset, long n_entry, LongVector ids, SWIGTYPE_p_unsigned_char code) { swigfaissJNI.OnDiskInvertedLists_update_entries(swigCPtr, this, list_no, offset, n_entry, SWIGTYPE_p_long_long.getCPtr(ids.data()), ids, SWIGTYPE_p_unsigned_char.getCPtr(code)); } public void resize(long list_no, long new_size) { swigfaissJNI.OnDiskInvertedLists_resize(swigCPtr, this, list_no, new_size); } public long merge_from(SWIGTYPE_p_p_faiss__InvertedLists ils, int n_il, boolean verbose) { return swigfaissJNI.OnDiskInvertedLists_merge_from__SWIG_0(swigCPtr, this, SWIGTYPE_p_p_faiss__InvertedLists.getCPtr(ils), n_il, verbose); } public long merge_from(SWIGTYPE_p_p_faiss__InvertedLists ils, int n_il) { return swigfaissJNI.OnDiskInvertedLists_merge_from__SWIG_1(swigCPtr, this, SWIGTYPE_p_p_faiss__InvertedLists.getCPtr(ils), n_il); } public long merge_from_1(InvertedLists il, boolean verbose) { return swigfaissJNI.OnDiskInvertedLists_merge_from_1__SWIG_0(swigCPtr, this, InvertedLists.getCPtr(il), il, verbose); } public long merge_from_1(InvertedLists il) { return swigfaissJNI.OnDiskInvertedLists_merge_from_1__SWIG_1(swigCPtr, this, InvertedLists.getCPtr(il), il); } public void crop_invlists(long l0, long l1) { swigfaissJNI.OnDiskInvertedLists_crop_invlists(swigCPtr, this, l0, l1); } public void prefetch_lists(LongVector list_nos, int nlist) { swigfaissJNI.OnDiskInvertedLists_prefetch_lists(swigCPtr, this, SWIGTYPE_p_long_long.getCPtr(list_nos.data()), list_nos, nlist); } public void setLocks(SWIGTYPE_p_faiss__LockLevels value) { swigfaissJNI.OnDiskInvertedLists_locks_set(swigCPtr, this, SWIGTYPE_p_faiss__LockLevels.getCPtr(value)); } public SWIGTYPE_p_faiss__LockLevels getLocks() { long cPtr = swigfaissJNI.OnDiskInvertedLists_locks_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_faiss__LockLevels(cPtr, false); } public void setPf(SWIGTYPE_p_faiss__OnDiskInvertedLists__OngoingPrefetch value) { swigfaissJNI.OnDiskInvertedLists_pf_set(swigCPtr, this, SWIGTYPE_p_faiss__OnDiskInvertedLists__OngoingPrefetch.getCPtr(value)); } public SWIGTYPE_p_faiss__OnDiskInvertedLists__OngoingPrefetch getPf() { long cPtr = swigfaissJNI.OnDiskInvertedLists_pf_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_faiss__OnDiskInvertedLists__OngoingPrefetch(cPtr, false); } public void setPrefetch_nthread(int value) { swigfaissJNI.OnDiskInvertedLists_prefetch_nthread_set(swigCPtr, this, value); } public int getPrefetch_nthread() { return swigfaissJNI.OnDiskInvertedLists_prefetch_nthread_get(swigCPtr, this); } public void do_mmap() { swigfaissJNI.OnDiskInvertedLists_do_mmap(swigCPtr, this); } public void update_totsize(long new_totsize) { swigfaissJNI.OnDiskInvertedLists_update_totsize(swigCPtr, this, new_totsize); } public void resize_locked(long list_no, long new_size) { swigfaissJNI.OnDiskInvertedLists_resize_locked(swigCPtr, this, list_no, new_size); } public long allocate_slot(long capacity) { return swigfaissJNI.OnDiskInvertedLists_allocate_slot(swigCPtr, this, capacity); } public void free_slot(long offset, long capacity) { swigfaissJNI.OnDiskInvertedLists_free_slot(swigCPtr, this, offset, capacity); } public void set_all_lists_sizes(SWIGTYPE_p_unsigned_long sizes) { swigfaissJNI.OnDiskInvertedLists_set_all_lists_sizes(swigCPtr, this, SWIGTYPE_p_unsigned_long.getCPtr(sizes)); } public OnDiskInvertedLists() { this(swigfaissJNI.new_OnDiskInvertedLists__SWIG_1(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/OnDiskInvertedListsIOHook.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class OnDiskInvertedListsIOHook { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected OnDiskInvertedListsIOHook(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(OnDiskInvertedListsIOHook obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_OnDiskInvertedListsIOHook(swigCPtr); } swigCPtr = 0; } } public OnDiskInvertedListsIOHook() { this(swigfaissJNI.new_OnDiskInvertedListsIOHook(), true); } public void write(InvertedLists ils, SWIGTYPE_p_IOWriter f) { swigfaissJNI.OnDiskInvertedListsIOHook_write(swigCPtr, this, InvertedLists.getCPtr(ils), ils, SWIGTYPE_p_IOWriter.getCPtr(f)); } public InvertedLists read(SWIGTYPE_p_IOReader f, int io_flags) { long cPtr = swigfaissJNI.OnDiskInvertedListsIOHook_read(swigCPtr, this, SWIGTYPE_p_IOReader.getCPtr(f), io_flags); return (cPtr == 0) ? null : new InvertedLists(cPtr, false); } public InvertedLists read_ArrayInvertedLists(SWIGTYPE_p_IOReader f, int io_flags, long nlist, long code_size, Uint64Vector sizes) { long cPtr = swigfaissJNI.OnDiskInvertedListsIOHook_read_ArrayInvertedLists(swigCPtr, this, SWIGTYPE_p_IOReader.getCPtr(f), io_flags, nlist, code_size, Uint64Vector.getCPtr(sizes), sizes); return (cPtr == 0) ? null : new InvertedLists(cPtr, false); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/OnDiskOneList.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class OnDiskOneList { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected OnDiskOneList(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(OnDiskOneList obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_OnDiskOneList(swigCPtr); } swigCPtr = 0; } } public void setSize(long value) { swigfaissJNI.OnDiskOneList_size_set(swigCPtr, this, value); } public long getSize() { return swigfaissJNI.OnDiskOneList_size_get(swigCPtr, this); } public void setCapacity(long value) { swigfaissJNI.OnDiskOneList_capacity_set(swigCPtr, this, value); } public long getCapacity() { return swigfaissJNI.OnDiskOneList_capacity_get(swigCPtr, this); } public void setOffset(long value) { swigfaissJNI.OnDiskOneList_offset_set(swigCPtr, this, value); } public long getOffset() { return swigfaissJNI.OnDiskOneList_offset_get(swigCPtr, this); } public OnDiskOneList() { this(swigfaissJNI.new_OnDiskOneList(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/OneRecallAtRCriterion.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class OneRecallAtRCriterion extends AutoTuneCriterion { private transient long swigCPtr; protected OneRecallAtRCriterion(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.OneRecallAtRCriterion_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(OneRecallAtRCriterion obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_OneRecallAtRCriterion(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setR(long value) { swigfaissJNI.OneRecallAtRCriterion_R_set(swigCPtr, this, value); } public long getR() { return swigfaissJNI.OneRecallAtRCriterion_R_get(swigCPtr, this); } public OneRecallAtRCriterion(long nq, long R) { this(swigfaissJNI.new_OneRecallAtRCriterion(nq, R), true); } public double evaluate(SWIGTYPE_p_float D, LongVector I) { return swigfaissJNI.OneRecallAtRCriterion_evaluate(swigCPtr, this, SWIGTYPE_p_float.getCPtr(D), SWIGTYPE_p_long_long.getCPtr(I.data()), I); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/OperatingPoint.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class OperatingPoint { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected OperatingPoint(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(OperatingPoint obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_OperatingPoint(swigCPtr); } swigCPtr = 0; } } public void setPerf(double value) { swigfaissJNI.OperatingPoint_perf_set(swigCPtr, this, value); } public double getPerf() { return swigfaissJNI.OperatingPoint_perf_get(swigCPtr, this); } public void setT(double value) { swigfaissJNI.OperatingPoint_t_set(swigCPtr, this, value); } public double getT() { return swigfaissJNI.OperatingPoint_t_get(swigCPtr, this); } public void setKey(String value) { swigfaissJNI.OperatingPoint_key_set(swigCPtr, this, value); } public String getKey() { return swigfaissJNI.OperatingPoint_key_get(swigCPtr, this); } public void setCno(long value) { swigfaissJNI.OperatingPoint_cno_set(swigCPtr, this, value); } public long getCno() { return swigfaissJNI.OperatingPoint_cno_get(swigCPtr, this); } public OperatingPoint() { this(swigfaissJNI.new_OperatingPoint(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/OperatingPointVector.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class OperatingPointVector { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected OperatingPointVector(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(OperatingPointVector obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_OperatingPointVector(swigCPtr); } swigCPtr = 0; } } public OperatingPointVector() { this(swigfaissJNI.new_OperatingPointVector(), true); } public void push_back(OperatingPoint arg0) { swigfaissJNI.OperatingPointVector_push_back(swigCPtr, this, OperatingPoint.getCPtr(arg0), arg0); } public void clear() { swigfaissJNI.OperatingPointVector_clear(swigCPtr, this); } public OperatingPoint data() { long cPtr = swigfaissJNI.OperatingPointVector_data(swigCPtr, this); return (cPtr == 0) ? null : new OperatingPoint(cPtr, false); } public long size() { return swigfaissJNI.OperatingPointVector_size(swigCPtr, this); } public OperatingPoint at(long n) { return new OperatingPoint(swigfaissJNI.OperatingPointVector_at(swigCPtr, this, n), true); } public void resize(long n) { swigfaissJNI.OperatingPointVector_resize(swigCPtr, this, n); } public void reserve(long n) { swigfaissJNI.OperatingPointVector_reserve(swigCPtr, this, n); } public void swap(OperatingPointVector other) { swigfaissJNI.OperatingPointVector_swap(swigCPtr, this, OperatingPointVector.getCPtr(other), other); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/OperatingPoints.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class OperatingPoints { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected OperatingPoints(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(OperatingPoints obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_OperatingPoints(swigCPtr); } swigCPtr = 0; } } public void setAll_pts(OperatingPointVector value) { swigfaissJNI.OperatingPoints_all_pts_set(swigCPtr, this, OperatingPointVector.getCPtr(value), value); } public OperatingPointVector getAll_pts() { long cPtr = swigfaissJNI.OperatingPoints_all_pts_get(swigCPtr, this); return (cPtr == 0) ? null : new OperatingPointVector(cPtr, false); } public void setOptimal_pts(OperatingPointVector value) { swigfaissJNI.OperatingPoints_optimal_pts_set(swigCPtr, this, OperatingPointVector.getCPtr(value), value); } public OperatingPointVector getOptimal_pts() { long cPtr = swigfaissJNI.OperatingPoints_optimal_pts_get(swigCPtr, this); return (cPtr == 0) ? null : new OperatingPointVector(cPtr, false); } public OperatingPoints() { this(swigfaissJNI.new_OperatingPoints(), true); } public int merge_with(OperatingPoints other, String prefix) { return swigfaissJNI.OperatingPoints_merge_with__SWIG_0(swigCPtr, this, OperatingPoints.getCPtr(other), other, prefix); } public int merge_with(OperatingPoints other) { return swigfaissJNI.OperatingPoints_merge_with__SWIG_1(swigCPtr, this, OperatingPoints.getCPtr(other), other); } public void clear() { swigfaissJNI.OperatingPoints_clear(swigCPtr, this); } public boolean add(double perf, double t, String key, long cno) { return swigfaissJNI.OperatingPoints_add__SWIG_0(swigCPtr, this, perf, t, key, cno); } public boolean add(double perf, double t, String key) { return swigfaissJNI.OperatingPoints_add__SWIG_1(swigCPtr, this, perf, t, key); } public double t_for_perf(double perf) { return swigfaissJNI.OperatingPoints_t_for_perf(swigCPtr, this, perf); } public void display(boolean only_optimal) { swigfaissJNI.OperatingPoints_display__SWIG_0(swigCPtr, this, only_optimal); } public void display() { swigfaissJNI.OperatingPoints_display__SWIG_1(swigCPtr, this); } public void all_to_gnuplot(String fname) { swigfaissJNI.OperatingPoints_all_to_gnuplot(swigCPtr, this, fname); } public void optimal_to_gnuplot(String fname) { swigfaissJNI.OperatingPoints_optimal_to_gnuplot(swigCPtr, this, fname); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/PCAMatrix.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class PCAMatrix extends LinearTransform { private transient long swigCPtr; protected PCAMatrix(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.PCAMatrix_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(PCAMatrix obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_PCAMatrix(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setEigen_power(float value) { swigfaissJNI.PCAMatrix_eigen_power_set(swigCPtr, this, value); } public float getEigen_power() { return swigfaissJNI.PCAMatrix_eigen_power_get(swigCPtr, this); } public void setEpsilon(float value) { swigfaissJNI.PCAMatrix_epsilon_set(swigCPtr, this, value); } public float getEpsilon() { return swigfaissJNI.PCAMatrix_epsilon_get(swigCPtr, this); } public void setRandom_rotation(boolean value) { swigfaissJNI.PCAMatrix_random_rotation_set(swigCPtr, this, value); } public boolean getRandom_rotation() { return swigfaissJNI.PCAMatrix_random_rotation_get(swigCPtr, this); } public void setMax_points_per_d(long value) { swigfaissJNI.PCAMatrix_max_points_per_d_set(swigCPtr, this, value); } public long getMax_points_per_d() { return swigfaissJNI.PCAMatrix_max_points_per_d_get(swigCPtr, this); } public void setBalanced_bins(int value) { swigfaissJNI.PCAMatrix_balanced_bins_set(swigCPtr, this, value); } public int getBalanced_bins() { return swigfaissJNI.PCAMatrix_balanced_bins_get(swigCPtr, this); } public void setMean(FloatVector value) { swigfaissJNI.PCAMatrix_mean_set(swigCPtr, this, FloatVector.getCPtr(value), value); } public FloatVector getMean() { long cPtr = swigfaissJNI.PCAMatrix_mean_get(swigCPtr, this); return (cPtr == 0) ? null : new FloatVector(cPtr, false); } public void setEigenvalues(FloatVector value) { swigfaissJNI.PCAMatrix_eigenvalues_set(swigCPtr, this, FloatVector.getCPtr(value), value); } public FloatVector getEigenvalues() { long cPtr = swigfaissJNI.PCAMatrix_eigenvalues_get(swigCPtr, this); return (cPtr == 0) ? null : new FloatVector(cPtr, false); } public void setPCAMat(FloatVector value) { swigfaissJNI.PCAMatrix_PCAMat_set(swigCPtr, this, FloatVector.getCPtr(value), value); } public FloatVector getPCAMat() { long cPtr = swigfaissJNI.PCAMatrix_PCAMat_get(swigCPtr, this); return (cPtr == 0) ? null : new FloatVector(cPtr, false); } public PCAMatrix(int d_in, int d_out, float eigen_power, boolean random_rotation) { this(swigfaissJNI.new_PCAMatrix__SWIG_0(d_in, d_out, eigen_power, random_rotation), true); } public PCAMatrix(int d_in, int d_out, float eigen_power) { this(swigfaissJNI.new_PCAMatrix__SWIG_1(d_in, d_out, eigen_power), true); } public PCAMatrix(int d_in, int d_out) { this(swigfaissJNI.new_PCAMatrix__SWIG_2(d_in, d_out), true); } public PCAMatrix(int d_in) { this(swigfaissJNI.new_PCAMatrix__SWIG_3(d_in), true); } public PCAMatrix() { this(swigfaissJNI.new_PCAMatrix__SWIG_4(), true); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.PCAMatrix_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public void copy_from(PCAMatrix other) { swigfaissJNI.PCAMatrix_copy_from(swigCPtr, this, PCAMatrix.getCPtr(other), other); } public void prepare_Ab() { swigfaissJNI.PCAMatrix_prepare_Ab(swigCPtr, this); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/PQDecoder16.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class PQDecoder16 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected PQDecoder16(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(PQDecoder16 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_PQDecoder16(swigCPtr); } swigCPtr = 0; } } public void setCode(SWIGTYPE_p_uint16_t value) { swigfaissJNI.PQDecoder16_code_set(swigCPtr, this, SWIGTYPE_p_uint16_t.getCPtr(value)); } public SWIGTYPE_p_uint16_t getCode() { long cPtr = swigfaissJNI.PQDecoder16_code_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_uint16_t(cPtr, false); } public PQDecoder16(SWIGTYPE_p_unsigned_char code, int nbits) { this(swigfaissJNI.new_PQDecoder16(SWIGTYPE_p_unsigned_char.getCPtr(code), nbits), true); } public long decode() { return swigfaissJNI.PQDecoder16_decode(swigCPtr, this); } public final static int nbits = swigfaissJNI.PQDecoder16_nbits_get(); } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/PQDecoder8.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class PQDecoder8 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected PQDecoder8(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(PQDecoder8 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_PQDecoder8(swigCPtr); } swigCPtr = 0; } } public void setCode(SWIGTYPE_p_unsigned_char value) { swigfaissJNI.PQDecoder8_code_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(value)); } public SWIGTYPE_p_unsigned_char getCode() { long cPtr = swigfaissJNI.PQDecoder8_code_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public PQDecoder8(SWIGTYPE_p_unsigned_char code, int nbits) { this(swigfaissJNI.new_PQDecoder8(SWIGTYPE_p_unsigned_char.getCPtr(code), nbits), true); } public long decode() { return swigfaissJNI.PQDecoder8_decode(swigCPtr, this); } public final static int nbits = swigfaissJNI.PQDecoder8_nbits_get(); } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/PQDecoderGeneric.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class PQDecoderGeneric { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected PQDecoderGeneric(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(PQDecoderGeneric obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_PQDecoderGeneric(swigCPtr); } swigCPtr = 0; } } public void setCode(SWIGTYPE_p_unsigned_char value) { swigfaissJNI.PQDecoderGeneric_code_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(value)); } public SWIGTYPE_p_unsigned_char getCode() { long cPtr = swigfaissJNI.PQDecoderGeneric_code_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public void setOffset(short value) { swigfaissJNI.PQDecoderGeneric_offset_set(swigCPtr, this, value); } public short getOffset() { return swigfaissJNI.PQDecoderGeneric_offset_get(swigCPtr, this); } public int getNbits() { return swigfaissJNI.PQDecoderGeneric_nbits_get(swigCPtr, this); } public long getMask() { return swigfaissJNI.PQDecoderGeneric_mask_get(swigCPtr, this); } public void setReg(short value) { swigfaissJNI.PQDecoderGeneric_reg_set(swigCPtr, this, value); } public short getReg() { return swigfaissJNI.PQDecoderGeneric_reg_get(swigCPtr, this); } public PQDecoderGeneric(SWIGTYPE_p_unsigned_char code, int nbits) { this(swigfaissJNI.new_PQDecoderGeneric(SWIGTYPE_p_unsigned_char.getCPtr(code), nbits), true); } public long decode() { return swigfaissJNI.PQDecoderGeneric_decode(swigCPtr, this); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/PQEncoder16.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class PQEncoder16 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected PQEncoder16(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(PQEncoder16 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_PQEncoder16(swigCPtr); } swigCPtr = 0; } } public void setCode(SWIGTYPE_p_uint16_t value) { swigfaissJNI.PQEncoder16_code_set(swigCPtr, this, SWIGTYPE_p_uint16_t.getCPtr(value)); } public SWIGTYPE_p_uint16_t getCode() { long cPtr = swigfaissJNI.PQEncoder16_code_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_uint16_t(cPtr, false); } public PQEncoder16(SWIGTYPE_p_unsigned_char code, int nbits) { this(swigfaissJNI.new_PQEncoder16(SWIGTYPE_p_unsigned_char.getCPtr(code), nbits), true); } public void encode(long x) { swigfaissJNI.PQEncoder16_encode(swigCPtr, this, x); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/PQEncoder8.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class PQEncoder8 { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected PQEncoder8(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(PQEncoder8 obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_PQEncoder8(swigCPtr); } swigCPtr = 0; } } public void setCode(SWIGTYPE_p_unsigned_char value) { swigfaissJNI.PQEncoder8_code_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(value)); } public SWIGTYPE_p_unsigned_char getCode() { long cPtr = swigfaissJNI.PQEncoder8_code_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public PQEncoder8(SWIGTYPE_p_unsigned_char code, int nbits) { this(swigfaissJNI.new_PQEncoder8(SWIGTYPE_p_unsigned_char.getCPtr(code), nbits), true); } public void encode(long x) { swigfaissJNI.PQEncoder8_encode(swigCPtr, this, x); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/PQEncoderGeneric.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class PQEncoderGeneric { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected PQEncoderGeneric(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(PQEncoderGeneric obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_PQEncoderGeneric(swigCPtr); } swigCPtr = 0; } } public void setCode(SWIGTYPE_p_unsigned_char value) { swigfaissJNI.PQEncoderGeneric_code_set(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(value)); } public SWIGTYPE_p_unsigned_char getCode() { long cPtr = swigfaissJNI.PQEncoderGeneric_code_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public void setOffset(short value) { swigfaissJNI.PQEncoderGeneric_offset_set(swigCPtr, this, value); } public short getOffset() { return swigfaissJNI.PQEncoderGeneric_offset_get(swigCPtr, this); } public int getNbits() { return swigfaissJNI.PQEncoderGeneric_nbits_get(swigCPtr, this); } public void setReg(short value) { swigfaissJNI.PQEncoderGeneric_reg_set(swigCPtr, this, value); } public short getReg() { return swigfaissJNI.PQEncoderGeneric_reg_get(swigCPtr, this); } public PQEncoderGeneric(SWIGTYPE_p_unsigned_char code, int nbits, short offset) { this(swigfaissJNI.new_PQEncoderGeneric__SWIG_0(SWIGTYPE_p_unsigned_char.getCPtr(code), nbits, offset), true); } public PQEncoderGeneric(SWIGTYPE_p_unsigned_char code, int nbits) { this(swigfaissJNI.new_PQEncoderGeneric__SWIG_1(SWIGTYPE_p_unsigned_char.getCPtr(code), nbits), true); } public void encode(long x) { swigfaissJNI.PQEncoderGeneric_encode(swigCPtr, this, x); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/ParameterRange.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class ParameterRange { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected ParameterRange(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(ParameterRange obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_ParameterRange(swigCPtr); } swigCPtr = 0; } } public void setName(String value) { swigfaissJNI.ParameterRange_name_set(swigCPtr, this, value); } public String getName() { return swigfaissJNI.ParameterRange_name_get(swigCPtr, this); } public void setValues(DoubleVector value) { swigfaissJNI.ParameterRange_values_set(swigCPtr, this, DoubleVector.getCPtr(value), value); } public DoubleVector getValues() { long cPtr = swigfaissJNI.ParameterRange_values_get(swigCPtr, this); return (cPtr == 0) ? null : new DoubleVector(cPtr, false); } public ParameterRange() { this(swigfaissJNI.new_ParameterRange(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/ParameterSpace.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class ParameterSpace { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected ParameterSpace(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(ParameterSpace obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_ParameterSpace(swigCPtr); } swigCPtr = 0; } } public void setParameter_ranges(SWIGTYPE_p_std__vectorT_faiss__ParameterRange_t value) { swigfaissJNI.ParameterSpace_parameter_ranges_set(swigCPtr, this, SWIGTYPE_p_std__vectorT_faiss__ParameterRange_t.getCPtr(value)); } public SWIGTYPE_p_std__vectorT_faiss__ParameterRange_t getParameter_ranges() { long cPtr = swigfaissJNI.ParameterSpace_parameter_ranges_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_faiss__ParameterRange_t(cPtr, false); } public void setVerbose(int value) { swigfaissJNI.ParameterSpace_verbose_set(swigCPtr, this, value); } public int getVerbose() { return swigfaissJNI.ParameterSpace_verbose_get(swigCPtr, this); } public void setN_experiments(int value) { swigfaissJNI.ParameterSpace_n_experiments_set(swigCPtr, this, value); } public int getN_experiments() { return swigfaissJNI.ParameterSpace_n_experiments_get(swigCPtr, this); } public void setBatchsize(long value) { swigfaissJNI.ParameterSpace_batchsize_set(swigCPtr, this, value); } public long getBatchsize() { return swigfaissJNI.ParameterSpace_batchsize_get(swigCPtr, this); } public void setThread_over_batches(boolean value) { swigfaissJNI.ParameterSpace_thread_over_batches_set(swigCPtr, this, value); } public boolean getThread_over_batches() { return swigfaissJNI.ParameterSpace_thread_over_batches_get(swigCPtr, this); } public void setMin_test_duration(double value) { swigfaissJNI.ParameterSpace_min_test_duration_set(swigCPtr, this, value); } public double getMin_test_duration() { return swigfaissJNI.ParameterSpace_min_test_duration_get(swigCPtr, this); } public ParameterSpace() { this(swigfaissJNI.new_ParameterSpace(), true); } public long n_combinations() { return swigfaissJNI.ParameterSpace_n_combinations(swigCPtr, this); } public boolean combination_ge(long c1, long c2) { return swigfaissJNI.ParameterSpace_combination_ge(swigCPtr, this, c1, c2); } public String combination_name(long cno) { return swigfaissJNI.ParameterSpace_combination_name(swigCPtr, this, cno); } public void display() { swigfaissJNI.ParameterSpace_display(swigCPtr, this); } public ParameterRange add_range(String name) { return new ParameterRange(swigfaissJNI.ParameterSpace_add_range(swigCPtr, this, name), false); } public void initialize(Index index) { swigfaissJNI.ParameterSpace_initialize(swigCPtr, this, Index.getCPtr(index), index); } public void set_index_parameters(Index index, long cno) { swigfaissJNI.ParameterSpace_set_index_parameters__SWIG_0(swigCPtr, this, Index.getCPtr(index), index, cno); } public void set_index_parameters(Index index, String param_string) { swigfaissJNI.ParameterSpace_set_index_parameters__SWIG_1(swigCPtr, this, Index.getCPtr(index), index, param_string); } public void set_index_parameter(Index index, String name, double val) { swigfaissJNI.ParameterSpace_set_index_parameter(swigCPtr, this, Index.getCPtr(index), index, name, val); } public void update_bounds(long cno, OperatingPoint op, SWIGTYPE_p_double upper_bound_perf, SWIGTYPE_p_double lower_bound_t) { swigfaissJNI.ParameterSpace_update_bounds(swigCPtr, this, cno, OperatingPoint.getCPtr(op), op, SWIGTYPE_p_double.getCPtr(upper_bound_perf), SWIGTYPE_p_double.getCPtr(lower_bound_t)); } public void explore(Index index, long nq, SWIGTYPE_p_float xq, AutoTuneCriterion crit, OperatingPoints ops) { swigfaissJNI.ParameterSpace_explore(swigCPtr, this, Index.getCPtr(index), index, nq, SWIGTYPE_p_float.getCPtr(xq), AutoTuneCriterion.getCPtr(crit), crit, OperatingPoints.getCPtr(ops), ops); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/PartitionStats.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class PartitionStats { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected PartitionStats(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(PartitionStats obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_PartitionStats(swigCPtr); } swigCPtr = 0; } } public void setBissect_cycles(long value) { swigfaissJNI.PartitionStats_bissect_cycles_set(swigCPtr, this, value); } public long getBissect_cycles() { return swigfaissJNI.PartitionStats_bissect_cycles_get(swigCPtr, this); } public void setCompress_cycles(long value) { swigfaissJNI.PartitionStats_compress_cycles_set(swigCPtr, this, value); } public long getCompress_cycles() { return swigfaissJNI.PartitionStats_compress_cycles_get(swigCPtr, this); } public PartitionStats() { this(swigfaissJNI.new_PartitionStats(), true); } public void reset() { swigfaissJNI.PartitionStats_reset(swigCPtr, this); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/PermutationObjective.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class PermutationObjective { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected PermutationObjective(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(PermutationObjective obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_PermutationObjective(swigCPtr); } swigCPtr = 0; } } public void setN(int value) { swigfaissJNI.PermutationObjective_n_set(swigCPtr, this, value); } public int getN() { return swigfaissJNI.PermutationObjective_n_get(swigCPtr, this); } public double compute_cost(SWIGTYPE_p_int perm) { return swigfaissJNI.PermutationObjective_compute_cost(swigCPtr, this, SWIGTYPE_p_int.getCPtr(perm)); } public double cost_update(SWIGTYPE_p_int perm, int iw, int jw) { return swigfaissJNI.PermutationObjective_cost_update(swigCPtr, this, SWIGTYPE_p_int.getCPtr(perm), iw, jw); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/PolysemousTraining.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class PolysemousTraining extends SimulatedAnnealingParameters { private transient long swigCPtr; protected PolysemousTraining(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.PolysemousTraining_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(PolysemousTraining obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_PolysemousTraining(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setOptimization_type(PolysemousTraining.Optimization_type_t value) { swigfaissJNI.PolysemousTraining_optimization_type_set(swigCPtr, this, value.swigValue()); } public PolysemousTraining.Optimization_type_t getOptimization_type() { return PolysemousTraining.Optimization_type_t.swigToEnum(swigfaissJNI.PolysemousTraining_optimization_type_get(swigCPtr, this)); } public void setNtrain_permutation(int value) { swigfaissJNI.PolysemousTraining_ntrain_permutation_set(swigCPtr, this, value); } public int getNtrain_permutation() { return swigfaissJNI.PolysemousTraining_ntrain_permutation_get(swigCPtr, this); } public void setDis_weight_factor(double value) { swigfaissJNI.PolysemousTraining_dis_weight_factor_set(swigCPtr, this, value); } public double getDis_weight_factor() { return swigfaissJNI.PolysemousTraining_dis_weight_factor_get(swigCPtr, this); } public void setMax_memory(long value) { swigfaissJNI.PolysemousTraining_max_memory_set(swigCPtr, this, value); } public long getMax_memory() { return swigfaissJNI.PolysemousTraining_max_memory_get(swigCPtr, this); } public void setLog_pattern(String value) { swigfaissJNI.PolysemousTraining_log_pattern_set(swigCPtr, this, value); } public String getLog_pattern() { return swigfaissJNI.PolysemousTraining_log_pattern_get(swigCPtr, this); } public PolysemousTraining() { this(swigfaissJNI.new_PolysemousTraining(), true); } public void optimize_pq_for_hamming(ProductQuantizer pq, long n, SWIGTYPE_p_float x) { swigfaissJNI.PolysemousTraining_optimize_pq_for_hamming(swigCPtr, this, ProductQuantizer.getCPtr(pq), pq, n, SWIGTYPE_p_float.getCPtr(x)); } public void optimize_ranking(ProductQuantizer pq, long n, SWIGTYPE_p_float x) { swigfaissJNI.PolysemousTraining_optimize_ranking(swigCPtr, this, ProductQuantizer.getCPtr(pq), pq, n, SWIGTYPE_p_float.getCPtr(x)); } public void optimize_reproduce_distances(ProductQuantizer pq) { swigfaissJNI.PolysemousTraining_optimize_reproduce_distances(swigCPtr, this, ProductQuantizer.getCPtr(pq), pq); } public long memory_usage_per_thread(ProductQuantizer pq) { return swigfaissJNI.PolysemousTraining_memory_usage_per_thread(swigCPtr, this, ProductQuantizer.getCPtr(pq), pq); } public final static class Optimization_type_t { public final static PolysemousTraining.Optimization_type_t OT_None = new PolysemousTraining.Optimization_type_t("OT_None"); public final static PolysemousTraining.Optimization_type_t OT_ReproduceDistances_affine = new PolysemousTraining.Optimization_type_t("OT_ReproduceDistances_affine"); public final static PolysemousTraining.Optimization_type_t OT_Ranking_weighted_diff = new PolysemousTraining.Optimization_type_t("OT_Ranking_weighted_diff"); public final int swigValue() { return swigValue; } public String toString() { return swigName; } public static Optimization_type_t swigToEnum(int swigValue) { if (swigValue < swigValues.length && swigValue >= 0 && swigValues[swigValue].swigValue == swigValue) return swigValues[swigValue]; for (int i = 0; i < swigValues.length; i++) if (swigValues[i].swigValue == swigValue) return swigValues[i]; throw new IllegalArgumentException("No enum " + Optimization_type_t.class + " with value " + swigValue); } private Optimization_type_t(String swigName) { this.swigName = swigName; this.swigValue = swigNext++; } private Optimization_type_t(String swigName, int swigValue) { this.swigName = swigName; this.swigValue = swigValue; swigNext = swigValue+1; } private Optimization_type_t(String swigName, Optimization_type_t swigEnum) { this.swigName = swigName; this.swigValue = swigEnum.swigValue; swigNext = this.swigValue+1; } private static Optimization_type_t[] swigValues = { OT_None, OT_ReproduceDistances_affine, OT_Ranking_weighted_diff }; private static int swigNext = 0; private final int swigValue; private final String swigName; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/ProductQuantizer.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class ProductQuantizer { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected ProductQuantizer(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(ProductQuantizer obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_ProductQuantizer(swigCPtr); } swigCPtr = 0; } } public void setD(long value) { swigfaissJNI.ProductQuantizer_d_set(swigCPtr, this, value); } public long getD() { return swigfaissJNI.ProductQuantizer_d_get(swigCPtr, this); } public void setM(long value) { swigfaissJNI.ProductQuantizer_M_set(swigCPtr, this, value); } public long getM() { return swigfaissJNI.ProductQuantizer_M_get(swigCPtr, this); } public void setNbits(long value) { swigfaissJNI.ProductQuantizer_nbits_set(swigCPtr, this, value); } public long getNbits() { return swigfaissJNI.ProductQuantizer_nbits_get(swigCPtr, this); } public void setDsub(long value) { swigfaissJNI.ProductQuantizer_dsub_set(swigCPtr, this, value); } public long getDsub() { return swigfaissJNI.ProductQuantizer_dsub_get(swigCPtr, this); } public void setCode_size(long value) { swigfaissJNI.ProductQuantizer_code_size_set(swigCPtr, this, value); } public long getCode_size() { return swigfaissJNI.ProductQuantizer_code_size_get(swigCPtr, this); } public void setKsub(long value) { swigfaissJNI.ProductQuantizer_ksub_set(swigCPtr, this, value); } public long getKsub() { return swigfaissJNI.ProductQuantizer_ksub_get(swigCPtr, this); } public void setVerbose(boolean value) { swigfaissJNI.ProductQuantizer_verbose_set(swigCPtr, this, value); } public boolean getVerbose() { return swigfaissJNI.ProductQuantizer_verbose_get(swigCPtr, this); } public void setTrain_type(ProductQuantizer.train_type_t value) { swigfaissJNI.ProductQuantizer_train_type_set(swigCPtr, this, value.swigValue()); } public ProductQuantizer.train_type_t getTrain_type() { return ProductQuantizer.train_type_t.swigToEnum(swigfaissJNI.ProductQuantizer_train_type_get(swigCPtr, this)); } public void setCp(ClusteringParameters value) { swigfaissJNI.ProductQuantizer_cp_set(swigCPtr, this, ClusteringParameters.getCPtr(value), value); } public ClusteringParameters getCp() { long cPtr = swigfaissJNI.ProductQuantizer_cp_get(swigCPtr, this); return (cPtr == 0) ? null : new ClusteringParameters(cPtr, false); } public void setAssign_index(Index value) { swigfaissJNI.ProductQuantizer_assign_index_set(swigCPtr, this, Index.getCPtr(value), value); } public Index getAssign_index() { long cPtr = swigfaissJNI.ProductQuantizer_assign_index_get(swigCPtr, this); return (cPtr == 0) ? null : new Index(cPtr, false); } public void setCentroids(FloatVector value) { swigfaissJNI.ProductQuantizer_centroids_set(swigCPtr, this, FloatVector.getCPtr(value), value); } public FloatVector getCentroids() { long cPtr = swigfaissJNI.ProductQuantizer_centroids_get(swigCPtr, this); return (cPtr == 0) ? null : new FloatVector(cPtr, false); } public SWIGTYPE_p_float get_centroids(long m, long i) { long cPtr = swigfaissJNI.ProductQuantizer_get_centroids(swigCPtr, this, m, i); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } public void train(int n, SWIGTYPE_p_float x) { swigfaissJNI.ProductQuantizer_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public ProductQuantizer(long d, long M, long nbits) { this(swigfaissJNI.new_ProductQuantizer__SWIG_0(d, M, nbits), true); } public ProductQuantizer() { this(swigfaissJNI.new_ProductQuantizer__SWIG_1(), true); } public void set_derived_values() { swigfaissJNI.ProductQuantizer_set_derived_values(swigCPtr, this); } public void set_params(SWIGTYPE_p_float centroids, int m) { swigfaissJNI.ProductQuantizer_set_params(swigCPtr, this, SWIGTYPE_p_float.getCPtr(centroids), m); } public void compute_code(SWIGTYPE_p_float x, SWIGTYPE_p_unsigned_char code) { swigfaissJNI.ProductQuantizer_compute_code(swigCPtr, this, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_unsigned_char.getCPtr(code)); } public void compute_codes(SWIGTYPE_p_float x, SWIGTYPE_p_unsigned_char codes, long n) { swigfaissJNI.ProductQuantizer_compute_codes(swigCPtr, this, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_unsigned_char.getCPtr(codes), n); } public void compute_codes_with_assign_index(SWIGTYPE_p_float x, SWIGTYPE_p_unsigned_char codes, long n) { swigfaissJNI.ProductQuantizer_compute_codes_with_assign_index(swigCPtr, this, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_unsigned_char.getCPtr(codes), n); } public void decode(SWIGTYPE_p_unsigned_char code, SWIGTYPE_p_float x) { swigfaissJNI.ProductQuantizer_decode__SWIG_0(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(code), SWIGTYPE_p_float.getCPtr(x)); } public void decode(SWIGTYPE_p_unsigned_char code, SWIGTYPE_p_float x, long n) { swigfaissJNI.ProductQuantizer_decode__SWIG_1(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(code), SWIGTYPE_p_float.getCPtr(x), n); } public void compute_code_from_distance_table(SWIGTYPE_p_float tab, SWIGTYPE_p_unsigned_char code) { swigfaissJNI.ProductQuantizer_compute_code_from_distance_table(swigCPtr, this, SWIGTYPE_p_float.getCPtr(tab), SWIGTYPE_p_unsigned_char.getCPtr(code)); } public void compute_distance_table(SWIGTYPE_p_float x, SWIGTYPE_p_float dis_table) { swigfaissJNI.ProductQuantizer_compute_distance_table(swigCPtr, this, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_float.getCPtr(dis_table)); } public void compute_inner_prod_table(SWIGTYPE_p_float x, SWIGTYPE_p_float dis_table) { swigfaissJNI.ProductQuantizer_compute_inner_prod_table(swigCPtr, this, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_float.getCPtr(dis_table)); } public void compute_distance_tables(long nx, SWIGTYPE_p_float x, SWIGTYPE_p_float dis_tables) { swigfaissJNI.ProductQuantizer_compute_distance_tables(swigCPtr, this, nx, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_float.getCPtr(dis_tables)); } public void compute_inner_prod_tables(long nx, SWIGTYPE_p_float x, SWIGTYPE_p_float dis_tables) { swigfaissJNI.ProductQuantizer_compute_inner_prod_tables(swigCPtr, this, nx, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_float.getCPtr(dis_tables)); } public void search(SWIGTYPE_p_float x, long nx, SWIGTYPE_p_unsigned_char codes, long ncodes, SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_float_int64_t_t_t res, boolean init_finalize_heap) { swigfaissJNI.ProductQuantizer_search__SWIG_0(swigCPtr, this, SWIGTYPE_p_float.getCPtr(x), nx, SWIGTYPE_p_unsigned_char.getCPtr(codes), ncodes, SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_float_int64_t_t_t.getCPtr(res), init_finalize_heap); } public void search(SWIGTYPE_p_float x, long nx, SWIGTYPE_p_unsigned_char codes, long ncodes, SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_float_int64_t_t_t res) { swigfaissJNI.ProductQuantizer_search__SWIG_1(swigCPtr, this, SWIGTYPE_p_float.getCPtr(x), nx, SWIGTYPE_p_unsigned_char.getCPtr(codes), ncodes, SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_float_int64_t_t_t.getCPtr(res)); } public void search_ip(SWIGTYPE_p_float x, long nx, SWIGTYPE_p_unsigned_char codes, long ncodes, SWIGTYPE_p_faiss__HeapArrayT_faiss__CMinT_float_int64_t_t_t res, boolean init_finalize_heap) { swigfaissJNI.ProductQuantizer_search_ip__SWIG_0(swigCPtr, this, SWIGTYPE_p_float.getCPtr(x), nx, SWIGTYPE_p_unsigned_char.getCPtr(codes), ncodes, SWIGTYPE_p_faiss__HeapArrayT_faiss__CMinT_float_int64_t_t_t.getCPtr(res), init_finalize_heap); } public void search_ip(SWIGTYPE_p_float x, long nx, SWIGTYPE_p_unsigned_char codes, long ncodes, SWIGTYPE_p_faiss__HeapArrayT_faiss__CMinT_float_int64_t_t_t res) { swigfaissJNI.ProductQuantizer_search_ip__SWIG_1(swigCPtr, this, SWIGTYPE_p_float.getCPtr(x), nx, SWIGTYPE_p_unsigned_char.getCPtr(codes), ncodes, SWIGTYPE_p_faiss__HeapArrayT_faiss__CMinT_float_int64_t_t_t.getCPtr(res)); } public void setSdc_table(FloatVector value) { swigfaissJNI.ProductQuantizer_sdc_table_set(swigCPtr, this, FloatVector.getCPtr(value), value); } public FloatVector getSdc_table() { long cPtr = swigfaissJNI.ProductQuantizer_sdc_table_get(swigCPtr, this); return (cPtr == 0) ? null : new FloatVector(cPtr, false); } public void compute_sdc_table() { swigfaissJNI.ProductQuantizer_compute_sdc_table(swigCPtr, this); } public void search_sdc(SWIGTYPE_p_unsigned_char qcodes, long nq, SWIGTYPE_p_unsigned_char bcodes, long ncodes, SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_float_int64_t_t_t res, boolean init_finalize_heap) { swigfaissJNI.ProductQuantizer_search_sdc__SWIG_0(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(qcodes), nq, SWIGTYPE_p_unsigned_char.getCPtr(bcodes), ncodes, SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_float_int64_t_t_t.getCPtr(res), init_finalize_heap); } public void search_sdc(SWIGTYPE_p_unsigned_char qcodes, long nq, SWIGTYPE_p_unsigned_char bcodes, long ncodes, SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_float_int64_t_t_t res) { swigfaissJNI.ProductQuantizer_search_sdc__SWIG_1(swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(qcodes), nq, SWIGTYPE_p_unsigned_char.getCPtr(bcodes), ncodes, SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_float_int64_t_t_t.getCPtr(res)); } public final static class train_type_t { public final static ProductQuantizer.train_type_t Train_default = new ProductQuantizer.train_type_t("Train_default"); public final static ProductQuantizer.train_type_t Train_hot_start = new ProductQuantizer.train_type_t("Train_hot_start"); public final static ProductQuantizer.train_type_t Train_shared = new ProductQuantizer.train_type_t("Train_shared"); public final static ProductQuantizer.train_type_t Train_hypercube = new ProductQuantizer.train_type_t("Train_hypercube"); public final static ProductQuantizer.train_type_t Train_hypercube_pca = new ProductQuantizer.train_type_t("Train_hypercube_pca"); public final int swigValue() { return swigValue; } public String toString() { return swigName; } public static train_type_t swigToEnum(int swigValue) { if (swigValue < swigValues.length && swigValue >= 0 && swigValues[swigValue].swigValue == swigValue) return swigValues[swigValue]; for (int i = 0; i < swigValues.length; i++) if (swigValues[i].swigValue == swigValue) return swigValues[i]; throw new IllegalArgumentException("No enum " + train_type_t.class + " with value " + swigValue); } private train_type_t(String swigName) { this.swigName = swigName; this.swigValue = swigNext++; } private train_type_t(String swigName, int swigValue) { this.swigName = swigName; this.swigValue = swigValue; swigNext = swigValue+1; } private train_type_t(String swigName, train_type_t swigEnum) { this.swigName = swigName; this.swigValue = swigEnum.swigValue; swigNext = this.swigValue+1; } private static train_type_t[] swigValues = { Train_default, Train_hot_start, Train_shared, Train_hypercube, Train_hypercube_pca }; private static int swigNext = 0; private final int swigValue; private final String swigName; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/ProgressiveDimClustering.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class ProgressiveDimClustering extends ProgressiveDimClusteringParameters { private transient long swigCPtr; protected ProgressiveDimClustering(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.ProgressiveDimClustering_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(ProgressiveDimClustering obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_ProgressiveDimClustering(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setD(long value) { swigfaissJNI.ProgressiveDimClustering_d_set(swigCPtr, this, value); } public long getD() { return swigfaissJNI.ProgressiveDimClustering_d_get(swigCPtr, this); } public void setK(long value) { swigfaissJNI.ProgressiveDimClustering_k_set(swigCPtr, this, value); } public long getK() { return swigfaissJNI.ProgressiveDimClustering_k_get(swigCPtr, this); } public void setCentroids(FloatVector value) { swigfaissJNI.ProgressiveDimClustering_centroids_set(swigCPtr, this, FloatVector.getCPtr(value), value); } public FloatVector getCentroids() { long cPtr = swigfaissJNI.ProgressiveDimClustering_centroids_get(swigCPtr, this); return (cPtr == 0) ? null : new FloatVector(cPtr, false); } public void setIteration_stats(SWIGTYPE_p_std__vectorT_faiss__ClusteringIterationStats_t value) { swigfaissJNI.ProgressiveDimClustering_iteration_stats_set(swigCPtr, this, SWIGTYPE_p_std__vectorT_faiss__ClusteringIterationStats_t.getCPtr(value)); } public SWIGTYPE_p_std__vectorT_faiss__ClusteringIterationStats_t getIteration_stats() { long cPtr = swigfaissJNI.ProgressiveDimClustering_iteration_stats_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_faiss__ClusteringIterationStats_t(cPtr, false); } public ProgressiveDimClustering(int d, int k) { this(swigfaissJNI.new_ProgressiveDimClustering__SWIG_0(d, k), true); } public ProgressiveDimClustering(int d, int k, ProgressiveDimClusteringParameters cp) { this(swigfaissJNI.new_ProgressiveDimClustering__SWIG_1(d, k, ProgressiveDimClusteringParameters.getCPtr(cp), cp), true); } public void train(long n, SWIGTYPE_p_float x, ProgressiveDimIndexFactory factory) { swigfaissJNI.ProgressiveDimClustering_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), ProgressiveDimIndexFactory.getCPtr(factory), factory); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/ProgressiveDimClusteringParameters.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class ProgressiveDimClusteringParameters extends ClusteringParameters { private transient long swigCPtr; protected ProgressiveDimClusteringParameters(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.ProgressiveDimClusteringParameters_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(ProgressiveDimClusteringParameters obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_ProgressiveDimClusteringParameters(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setProgressive_dim_steps(int value) { swigfaissJNI.ProgressiveDimClusteringParameters_progressive_dim_steps_set(swigCPtr, this, value); } public int getProgressive_dim_steps() { return swigfaissJNI.ProgressiveDimClusteringParameters_progressive_dim_steps_get(swigCPtr, this); } public void setApply_pca(boolean value) { swigfaissJNI.ProgressiveDimClusteringParameters_apply_pca_set(swigCPtr, this, value); } public boolean getApply_pca() { return swigfaissJNI.ProgressiveDimClusteringParameters_apply_pca_get(swigCPtr, this); } public ProgressiveDimClusteringParameters() { this(swigfaissJNI.new_ProgressiveDimClusteringParameters(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/ProgressiveDimIndexFactory.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class ProgressiveDimIndexFactory { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected ProgressiveDimIndexFactory(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(ProgressiveDimIndexFactory obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_ProgressiveDimIndexFactory(swigCPtr); } swigCPtr = 0; } } public ProgressiveDimIndexFactory() { this(swigfaissJNI.new_ProgressiveDimIndexFactory(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/RandomRotationMatrix.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class RandomRotationMatrix extends LinearTransform { private transient long swigCPtr; protected RandomRotationMatrix(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.RandomRotationMatrix_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(RandomRotationMatrix obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_RandomRotationMatrix(swigCPtr); } swigCPtr = 0; } super.delete(); } public RandomRotationMatrix(int d_in, int d_out) { this(swigfaissJNI.new_RandomRotationMatrix__SWIG_0(d_in, d_out), true); } public void init(int seed) { swigfaissJNI.RandomRotationMatrix_init(swigCPtr, this, seed); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.RandomRotationMatrix_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public RandomRotationMatrix() { this(swigfaissJNI.new_RandomRotationMatrix__SWIG_1(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/RangeQueryResult.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class RangeQueryResult { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected RangeQueryResult(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(RangeQueryResult obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_RangeQueryResult(swigCPtr); } swigCPtr = 0; } } public void setQno(long value) { swigfaissJNI.RangeQueryResult_qno_set(swigCPtr, this, value); } public long getQno() { return swigfaissJNI.RangeQueryResult_qno_get(swigCPtr, this); } public void setNres(long value) { swigfaissJNI.RangeQueryResult_nres_set(swigCPtr, this, value); } public long getNres() { return swigfaissJNI.RangeQueryResult_nres_get(swigCPtr, this); } public void setPres(RangeSearchPartialResult value) { swigfaissJNI.RangeQueryResult_pres_set(swigCPtr, this, RangeSearchPartialResult.getCPtr(value), value); } public RangeSearchPartialResult getPres() { long cPtr = swigfaissJNI.RangeQueryResult_pres_get(swigCPtr, this); return (cPtr == 0) ? null : new RangeSearchPartialResult(cPtr, false); } public void add(float dis, long id) { swigfaissJNI.RangeQueryResult_add(swigCPtr, this, dis, id); } public RangeQueryResult() { this(swigfaissJNI.new_RangeQueryResult(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/RangeSearchPartialResult.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class RangeSearchPartialResult extends BufferList { private transient long swigCPtr; protected RangeSearchPartialResult(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.RangeSearchPartialResult_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(RangeSearchPartialResult obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_RangeSearchPartialResult(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setRes(RangeSearchResult value) { swigfaissJNI.RangeSearchPartialResult_res_set(swigCPtr, this, RangeSearchResult.getCPtr(value), value); } public RangeSearchResult getRes() { long cPtr = swigfaissJNI.RangeSearchPartialResult_res_get(swigCPtr, this); return (cPtr == 0) ? null : new RangeSearchResult(cPtr, false); } public void setQueries(SWIGTYPE_p_std__vectorT_faiss__RangeQueryResult_t value) { swigfaissJNI.RangeSearchPartialResult_queries_set(swigCPtr, this, SWIGTYPE_p_std__vectorT_faiss__RangeQueryResult_t.getCPtr(value)); } public SWIGTYPE_p_std__vectorT_faiss__RangeQueryResult_t getQueries() { long cPtr = swigfaissJNI.RangeSearchPartialResult_queries_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_faiss__RangeQueryResult_t(cPtr, false); } public RangeQueryResult new_result(long qno) { return new RangeQueryResult(swigfaissJNI.RangeSearchPartialResult_new_result(swigCPtr, this, qno), false); } public void set_lims() { swigfaissJNI.RangeSearchPartialResult_set_lims(swigCPtr, this); } public void copy_result(boolean incremental) { swigfaissJNI.RangeSearchPartialResult_copy_result__SWIG_0(swigCPtr, this, incremental); } public void copy_result() { swigfaissJNI.RangeSearchPartialResult_copy_result__SWIG_1(swigCPtr, this); } public static void merge(SWIGTYPE_p_std__vectorT_faiss__RangeSearchPartialResult_p_t partial_results, boolean do_delete) { swigfaissJNI.RangeSearchPartialResult_merge__SWIG_0(SWIGTYPE_p_std__vectorT_faiss__RangeSearchPartialResult_p_t.getCPtr(partial_results), do_delete); } public static void merge(SWIGTYPE_p_std__vectorT_faiss__RangeSearchPartialResult_p_t partial_results) { swigfaissJNI.RangeSearchPartialResult_merge__SWIG_1(SWIGTYPE_p_std__vectorT_faiss__RangeSearchPartialResult_p_t.getCPtr(partial_results)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/RangeSearchResult.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class RangeSearchResult { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected RangeSearchResult(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(RangeSearchResult obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_RangeSearchResult(swigCPtr); } swigCPtr = 0; } } public void setNq(long value) { swigfaissJNI.RangeSearchResult_nq_set(swigCPtr, this, value); } public long getNq() { return swigfaissJNI.RangeSearchResult_nq_get(swigCPtr, this); } public void setLims(SWIGTYPE_p_unsigned_long value) { swigfaissJNI.RangeSearchResult_lims_set(swigCPtr, this, SWIGTYPE_p_unsigned_long.getCPtr(value)); } public SWIGTYPE_p_unsigned_long getLims() { long cPtr = swigfaissJNI.RangeSearchResult_lims_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_long(cPtr, false); } public void setLabels(LongVector value) { swigfaissJNI.RangeSearchResult_labels_set(swigCPtr, this, SWIGTYPE_p_long_long.getCPtr(value.data()), value); } public LongVector getLabels() { return new LongVector(swigfaissJNI.RangeSearchResult_labels_get(swigCPtr, this), false); } public void setDistances(SWIGTYPE_p_float value) { swigfaissJNI.RangeSearchResult_distances_set(swigCPtr, this, SWIGTYPE_p_float.getCPtr(value)); } public SWIGTYPE_p_float getDistances() { long cPtr = swigfaissJNI.RangeSearchResult_distances_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } public void setBuffer_size(long value) { swigfaissJNI.RangeSearchResult_buffer_size_set(swigCPtr, this, value); } public long getBuffer_size() { return swigfaissJNI.RangeSearchResult_buffer_size_get(swigCPtr, this); } public void do_allocation() { swigfaissJNI.RangeSearchResult_do_allocation(swigCPtr, this); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/ReadOnlyInvertedLists.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class ReadOnlyInvertedLists extends InvertedLists { private transient long swigCPtr; protected ReadOnlyInvertedLists(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.ReadOnlyInvertedLists_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(ReadOnlyInvertedLists obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_ReadOnlyInvertedLists(swigCPtr); } swigCPtr = 0; } super.delete(); } public long add_entries(long list_no, long n_entry, LongVector ids, SWIGTYPE_p_unsigned_char code) { return swigfaissJNI.ReadOnlyInvertedLists_add_entries(swigCPtr, this, list_no, n_entry, SWIGTYPE_p_long_long.getCPtr(ids.data()), ids, SWIGTYPE_p_unsigned_char.getCPtr(code)); } public void update_entries(long list_no, long offset, long n_entry, LongVector ids, SWIGTYPE_p_unsigned_char code) { swigfaissJNI.ReadOnlyInvertedLists_update_entries(swigCPtr, this, list_no, offset, n_entry, SWIGTYPE_p_long_long.getCPtr(ids.data()), ids, SWIGTYPE_p_unsigned_char.getCPtr(code)); } public void resize(long list_no, long new_size) { swigfaissJNI.ReadOnlyInvertedLists_resize(swigCPtr, this, list_no, new_size); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/ReconstructFromNeighbors.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class ReconstructFromNeighbors { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected ReconstructFromNeighbors(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(ReconstructFromNeighbors obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_ReconstructFromNeighbors(swigCPtr); } swigCPtr = 0; } } public IndexHNSW getIndex() { return new IndexHNSW(swigfaissJNI.ReconstructFromNeighbors_index_get(swigCPtr, this), false); } public void setM(long value) { swigfaissJNI.ReconstructFromNeighbors_M_set(swigCPtr, this, value); } public long getM() { return swigfaissJNI.ReconstructFromNeighbors_M_get(swigCPtr, this); } public void setK(long value) { swigfaissJNI.ReconstructFromNeighbors_k_set(swigCPtr, this, value); } public long getK() { return swigfaissJNI.ReconstructFromNeighbors_k_get(swigCPtr, this); } public void setNsq(long value) { swigfaissJNI.ReconstructFromNeighbors_nsq_set(swigCPtr, this, value); } public long getNsq() { return swigfaissJNI.ReconstructFromNeighbors_nsq_get(swigCPtr, this); } public void setCode_size(long value) { swigfaissJNI.ReconstructFromNeighbors_code_size_set(swigCPtr, this, value); } public long getCode_size() { return swigfaissJNI.ReconstructFromNeighbors_code_size_get(swigCPtr, this); } public void setK_reorder(int value) { swigfaissJNI.ReconstructFromNeighbors_k_reorder_set(swigCPtr, this, value); } public int getK_reorder() { return swigfaissJNI.ReconstructFromNeighbors_k_reorder_get(swigCPtr, this); } public void setCodebook(FloatVector value) { swigfaissJNI.ReconstructFromNeighbors_codebook_set(swigCPtr, this, FloatVector.getCPtr(value), value); } public FloatVector getCodebook() { long cPtr = swigfaissJNI.ReconstructFromNeighbors_codebook_get(swigCPtr, this); return (cPtr == 0) ? null : new FloatVector(cPtr, false); } public void setCodes(ByteVector value) { swigfaissJNI.ReconstructFromNeighbors_codes_set(swigCPtr, this, ByteVector.getCPtr(value), value); } public ByteVector getCodes() { long cPtr = swigfaissJNI.ReconstructFromNeighbors_codes_get(swigCPtr, this); return (cPtr == 0) ? null : new ByteVector(cPtr, false); } public void setNtotal(long value) { swigfaissJNI.ReconstructFromNeighbors_ntotal_set(swigCPtr, this, value); } public long getNtotal() { return swigfaissJNI.ReconstructFromNeighbors_ntotal_get(swigCPtr, this); } public void setD(long value) { swigfaissJNI.ReconstructFromNeighbors_d_set(swigCPtr, this, value); } public long getD() { return swigfaissJNI.ReconstructFromNeighbors_d_get(swigCPtr, this); } public void setDsub(long value) { swigfaissJNI.ReconstructFromNeighbors_dsub_set(swigCPtr, this, value); } public long getDsub() { return swigfaissJNI.ReconstructFromNeighbors_dsub_get(swigCPtr, this); } public ReconstructFromNeighbors(IndexHNSW index, long k, long nsq) { this(swigfaissJNI.new_ReconstructFromNeighbors__SWIG_0(IndexHNSW.getCPtr(index), index, k, nsq), true); } public ReconstructFromNeighbors(IndexHNSW index, long k) { this(swigfaissJNI.new_ReconstructFromNeighbors__SWIG_1(IndexHNSW.getCPtr(index), index, k), true); } public ReconstructFromNeighbors(IndexHNSW index) { this(swigfaissJNI.new_ReconstructFromNeighbors__SWIG_2(IndexHNSW.getCPtr(index), index), true); } public void add_codes(long n, SWIGTYPE_p_float x) { swigfaissJNI.ReconstructFromNeighbors_add_codes(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public long compute_distances(long n, LongVector shortlist, SWIGTYPE_p_float query, SWIGTYPE_p_float distances) { return swigfaissJNI.ReconstructFromNeighbors_compute_distances(swigCPtr, this, n, SWIGTYPE_p_long_long.getCPtr(shortlist.data()), shortlist, SWIGTYPE_p_float.getCPtr(query), SWIGTYPE_p_float.getCPtr(distances)); } public void estimate_code(SWIGTYPE_p_float x, int i, SWIGTYPE_p_unsigned_char code) { swigfaissJNI.ReconstructFromNeighbors_estimate_code(swigCPtr, this, SWIGTYPE_p_float.getCPtr(x), i, SWIGTYPE_p_unsigned_char.getCPtr(code)); } public void reconstruct(int i, SWIGTYPE_p_float x, SWIGTYPE_p_float tmp) { swigfaissJNI.ReconstructFromNeighbors_reconstruct(swigCPtr, this, i, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_float.getCPtr(tmp)); } public void reconstruct_n(int n0, int ni, SWIGTYPE_p_float x) { swigfaissJNI.ReconstructFromNeighbors_reconstruct_n(swigCPtr, this, n0, ni, SWIGTYPE_p_float.getCPtr(x)); } public void get_neighbor_table(int i, SWIGTYPE_p_float out) { swigfaissJNI.ReconstructFromNeighbors_get_neighbor_table(swigCPtr, this, i, SWIGTYPE_p_float.getCPtr(out)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/RemapDimensionsTransform.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class RemapDimensionsTransform extends VectorTransform { private transient long swigCPtr; protected RemapDimensionsTransform(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.RemapDimensionsTransform_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(RemapDimensionsTransform obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_RemapDimensionsTransform(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setMap(IntVector value) { swigfaissJNI.RemapDimensionsTransform_map_set(swigCPtr, this, IntVector.getCPtr(value), value); } public IntVector getMap() { long cPtr = swigfaissJNI.RemapDimensionsTransform_map_get(swigCPtr, this); return (cPtr == 0) ? null : new IntVector(cPtr, false); } public RemapDimensionsTransform(int d_in, int d_out, SWIGTYPE_p_int map) { this(swigfaissJNI.new_RemapDimensionsTransform__SWIG_0(d_in, d_out, SWIGTYPE_p_int.getCPtr(map)), true); } public RemapDimensionsTransform(int d_in, int d_out, boolean uniform) { this(swigfaissJNI.new_RemapDimensionsTransform__SWIG_1(d_in, d_out, uniform), true); } public RemapDimensionsTransform(int d_in, int d_out) { this(swigfaissJNI.new_RemapDimensionsTransform__SWIG_2(d_in, d_out), true); } public void apply_noalloc(long n, SWIGTYPE_p_float x, SWIGTYPE_p_float xt) { swigfaissJNI.RemapDimensionsTransform_apply_noalloc(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_float.getCPtr(xt)); } public void reverse_transform(long n, SWIGTYPE_p_float xt, SWIGTYPE_p_float x) { swigfaissJNI.RemapDimensionsTransform_reverse_transform(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(xt), SWIGTYPE_p_float.getCPtr(x)); } public RemapDimensionsTransform() { this(swigfaissJNI.new_RemapDimensionsTransform__SWIG_3(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/ReproduceDistancesObjective.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class ReproduceDistancesObjective extends PermutationObjective { private transient long swigCPtr; protected ReproduceDistancesObjective(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.ReproduceDistancesObjective_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(ReproduceDistancesObjective obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_ReproduceDistancesObjective(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setDis_weight_factor(double value) { swigfaissJNI.ReproduceDistancesObjective_dis_weight_factor_set(swigCPtr, this, value); } public double getDis_weight_factor() { return swigfaissJNI.ReproduceDistancesObjective_dis_weight_factor_get(swigCPtr, this); } public static double sqr(double x) { return swigfaissJNI.ReproduceDistancesObjective_sqr(x); } public double dis_weight(double x) { return swigfaissJNI.ReproduceDistancesObjective_dis_weight(swigCPtr, this, x); } public void setSource_dis(DoubleVector value) { swigfaissJNI.ReproduceDistancesObjective_source_dis_set(swigCPtr, this, DoubleVector.getCPtr(value), value); } public DoubleVector getSource_dis() { long cPtr = swigfaissJNI.ReproduceDistancesObjective_source_dis_get(swigCPtr, this); return (cPtr == 0) ? null : new DoubleVector(cPtr, false); } public void setTarget_dis(SWIGTYPE_p_double value) { swigfaissJNI.ReproduceDistancesObjective_target_dis_set(swigCPtr, this, SWIGTYPE_p_double.getCPtr(value)); } public SWIGTYPE_p_double getTarget_dis() { long cPtr = swigfaissJNI.ReproduceDistancesObjective_target_dis_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_double(cPtr, false); } public void setWeights(DoubleVector value) { swigfaissJNI.ReproduceDistancesObjective_weights_set(swigCPtr, this, DoubleVector.getCPtr(value), value); } public DoubleVector getWeights() { long cPtr = swigfaissJNI.ReproduceDistancesObjective_weights_get(swigCPtr, this); return (cPtr == 0) ? null : new DoubleVector(cPtr, false); } public double get_source_dis(int i, int j) { return swigfaissJNI.ReproduceDistancesObjective_get_source_dis(swigCPtr, this, i, j); } public double compute_cost(SWIGTYPE_p_int perm) { return swigfaissJNI.ReproduceDistancesObjective_compute_cost(swigCPtr, this, SWIGTYPE_p_int.getCPtr(perm)); } public double cost_update(SWIGTYPE_p_int perm, int iw, int jw) { return swigfaissJNI.ReproduceDistancesObjective_cost_update(swigCPtr, this, SWIGTYPE_p_int.getCPtr(perm), iw, jw); } public ReproduceDistancesObjective(int n, SWIGTYPE_p_double source_dis_in, SWIGTYPE_p_double target_dis_in, double dis_weight_factor) { this(swigfaissJNI.new_ReproduceDistancesObjective(n, SWIGTYPE_p_double.getCPtr(source_dis_in), SWIGTYPE_p_double.getCPtr(target_dis_in), dis_weight_factor), true); } public static void compute_mean_stdev(SWIGTYPE_p_double tab, long n2, SWIGTYPE_p_double mean_out, SWIGTYPE_p_double stddev_out) { swigfaissJNI.ReproduceDistancesObjective_compute_mean_stdev(SWIGTYPE_p_double.getCPtr(tab), n2, SWIGTYPE_p_double.getCPtr(mean_out), SWIGTYPE_p_double.getCPtr(stddev_out)); } public void set_affine_target_dis(SWIGTYPE_p_double source_dis_in) { swigfaissJNI.ReproduceDistancesObjective_set_affine_target_dis(swigCPtr, this, SWIGTYPE_p_double.getCPtr(source_dis_in)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_AlignedTableT_float_32_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_AlignedTableT_float_32_t { private transient long swigCPtr; protected SWIGTYPE_p_AlignedTableT_float_32_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_AlignedTableT_float_32_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_AlignedTableT_float_32_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_AlignedTableT_float_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_AlignedTableT_float_t { private transient long swigCPtr; protected SWIGTYPE_p_AlignedTableT_float_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_AlignedTableT_float_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_AlignedTableT_float_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_DirectMap.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_DirectMap { private transient long swigCPtr; protected SWIGTYPE_p_DirectMap(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_DirectMap() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_DirectMap obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_DirectMap__Type.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_DirectMap__Type { private transient long swigCPtr; protected SWIGTYPE_p_DirectMap__Type(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_DirectMap__Type() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_DirectMap__Type obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_FILE.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_FILE { private transient long swigCPtr; protected SWIGTYPE_p_FILE(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_FILE() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_FILE obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_IOReader.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_IOReader { private transient long swigCPtr; protected SWIGTYPE_p_IOReader(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_IOReader() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_IOReader obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_IOWriter.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_IOWriter { private transient long swigCPtr; protected SWIGTYPE_p_IOWriter(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_IOWriter() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_IOWriter obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_ScalarQuantizer.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_ScalarQuantizer { private transient long swigCPtr; protected SWIGTYPE_p_ScalarQuantizer(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_ScalarQuantizer() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_ScalarQuantizer obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_ScalarQuantizer__QuantizerType.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_ScalarQuantizer__QuantizerType { private transient long swigCPtr; protected SWIGTYPE_p_ScalarQuantizer__QuantizerType(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_ScalarQuantizer__QuantizerType() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_ScalarQuantizer__QuantizerType obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_double.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_double { private transient long swigCPtr; protected SWIGTYPE_p_double(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_double() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_double obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_faiss__AlignedTableTightAllocT_float_32_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_faiss__AlignedTableTightAllocT_float_32_t { private transient long swigCPtr; protected SWIGTYPE_p_faiss__AlignedTableTightAllocT_float_32_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_faiss__AlignedTableTightAllocT_float_32_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_faiss__AlignedTableTightAllocT_float_32_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_faiss__AlignedTableTightAllocT_uint16_t_32_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_faiss__AlignedTableTightAllocT_uint16_t_32_t { private transient long swigCPtr; protected SWIGTYPE_p_faiss__AlignedTableTightAllocT_uint16_t_32_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_faiss__AlignedTableTightAllocT_uint16_t_32_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_faiss__AlignedTableTightAllocT_uint16_t_32_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_faiss__AlignedTableTightAllocT_unsigned_char_32_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_faiss__AlignedTableTightAllocT_unsigned_char_32_t { private transient long swigCPtr; protected SWIGTYPE_p_faiss__AlignedTableTightAllocT_unsigned_char_32_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_faiss__AlignedTableTightAllocT_unsigned_char_32_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_faiss__AlignedTableTightAllocT_unsigned_char_32_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_faiss__BinaryInvertedListScanner.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_faiss__BinaryInvertedListScanner { private transient long swigCPtr; protected SWIGTYPE_p_faiss__BinaryInvertedListScanner(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_faiss__BinaryInvertedListScanner() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_faiss__BinaryInvertedListScanner obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_float_int64_t_t_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_float_int64_t_t_t { private transient long swigCPtr; protected SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_float_int64_t_t_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_float_int64_t_t_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_float_int64_t_t_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_int_int64_t_t_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_int_int64_t_t_t { private transient long swigCPtr; protected SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_int_int64_t_t_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_int_int64_t_t_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_int_int64_t_t_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_faiss__HeapArrayT_faiss__CMinT_float_int64_t_t_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_faiss__HeapArrayT_faiss__CMinT_float_int64_t_t_t { private transient long swigCPtr; protected SWIGTYPE_p_faiss__HeapArrayT_faiss__CMinT_float_int64_t_t_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_faiss__HeapArrayT_faiss__CMinT_float_int64_t_t_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_faiss__HeapArrayT_faiss__CMinT_float_int64_t_t_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_faiss__IOReader.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_faiss__IOReader { private transient long swigCPtr; protected SWIGTYPE_p_faiss__IOReader(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_faiss__IOReader() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_faiss__IOReader obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_faiss__IOWriter.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_faiss__IOWriter { private transient long swigCPtr; protected SWIGTYPE_p_faiss__IOWriter(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_faiss__IOWriter() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_faiss__IOWriter obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_faiss__InvertedListScanner.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_faiss__InvertedListScanner { private transient long swigCPtr; protected SWIGTYPE_p_faiss__InvertedListScanner(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_faiss__InvertedListScanner() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_faiss__InvertedListScanner obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_faiss__LockLevels.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_faiss__LockLevels { private transient long swigCPtr; protected SWIGTYPE_p_faiss__LockLevels(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_faiss__LockLevels() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_faiss__LockLevels obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_faiss__OnDiskInvertedLists__OngoingPrefetch.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_faiss__OnDiskInvertedLists__OngoingPrefetch { private transient long swigCPtr; protected SWIGTYPE_p_faiss__OnDiskInvertedLists__OngoingPrefetch(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_faiss__OnDiskInvertedLists__OngoingPrefetch() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_faiss__OnDiskInvertedLists__OngoingPrefetch obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_faiss__RandomGenerator.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_faiss__RandomGenerator { private transient long swigCPtr; protected SWIGTYPE_p_faiss__RandomGenerator(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_faiss__RandomGenerator() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_faiss__RandomGenerator obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_float.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_float { private transient long swigCPtr; protected SWIGTYPE_p_float(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_float() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_float obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_int.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_int { private transient long swigCPtr; protected SWIGTYPE_p_int(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_int() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_int obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_long.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_long { private transient long swigCPtr; protected SWIGTYPE_p_long(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_long() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_long obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_long_long.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_long_long { private transient long swigCPtr; protected SWIGTYPE_p_long_long(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_long_long() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_long_long obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_omp_lock_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_omp_lock_t { private transient long swigCPtr; protected SWIGTYPE_p_omp_lock_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_omp_lock_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_omp_lock_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_p_faiss__Index.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_p_faiss__Index { private transient long swigCPtr; protected SWIGTYPE_p_p_faiss__Index(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_p_faiss__Index() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_p_faiss__Index obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_p_faiss__InvertedLists.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_p_faiss__InvertedLists { private transient long swigCPtr; protected SWIGTYPE_p_p_faiss__InvertedLists(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_p_faiss__InvertedLists() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_p_faiss__InvertedLists obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_p_faiss__VectorTransform.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_p_faiss__VectorTransform { private transient long swigCPtr; protected SWIGTYPE_p_p_faiss__VectorTransform(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_p_faiss__VectorTransform() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_p_faiss__VectorTransform obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__listT_faiss__OnDiskInvertedLists__Slot_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__listT_faiss__OnDiskInvertedLists__Slot_t { private transient long swigCPtr; protected SWIGTYPE_p_std__listT_faiss__OnDiskInvertedLists__Slot_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__listT_faiss__OnDiskInvertedLists__Slot_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__listT_faiss__OnDiskInvertedLists__Slot_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__pairT_float_int_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__pairT_float_int_t { private transient long swigCPtr; protected SWIGTYPE_p_std__pairT_float_int_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__pairT_float_int_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__pairT_float_int_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__priority_queueT_faiss__HNSW__NodeDistFarther_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__priority_queueT_faiss__HNSW__NodeDistFarther_t { private transient long swigCPtr; protected SWIGTYPE_p_std__priority_queueT_faiss__HNSW__NodeDistFarther_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__priority_queueT_faiss__HNSW__NodeDistFarther_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__priority_queueT_faiss__HNSW__NodeDistFarther_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__priority_queueT_std__pairT_float_int_t_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__priority_queueT_std__pairT_float_int_t_t { private transient long swigCPtr; protected SWIGTYPE_p_std__priority_queueT_std__pairT_float_int_t_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__priority_queueT_std__pairT_float_int_t_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__priority_queueT_std__pairT_float_int_t_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__unordered_mapT_long_long_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__unordered_mapT_long_long_t { private transient long swigCPtr; protected SWIGTYPE_p_std__unordered_mapT_long_long_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__unordered_mapT_long_long_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__unordered_mapT_long_long_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__unordered_multimapT_int64_t_int64_t_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__unordered_multimapT_int64_t_int64_t_t { private transient long swigCPtr; protected SWIGTYPE_p_std__unordered_multimapT_int64_t_int64_t_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__unordered_multimapT_int64_t_int64_t_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__unordered_multimapT_int64_t_int64_t_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__vectorT_faiss__BufferList__Buffer_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__vectorT_faiss__BufferList__Buffer_t { private transient long swigCPtr; protected SWIGTYPE_p_std__vectorT_faiss__BufferList__Buffer_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__vectorT_faiss__BufferList__Buffer_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__vectorT_faiss__BufferList__Buffer_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__vectorT_faiss__ClusteringIterationStats_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__vectorT_faiss__ClusteringIterationStats_t { private transient long swigCPtr; protected SWIGTYPE_p_std__vectorT_faiss__ClusteringIterationStats_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__vectorT_faiss__ClusteringIterationStats_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__vectorT_faiss__ClusteringIterationStats_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__vectorT_faiss__HNSW__NodeDistFarther_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__vectorT_faiss__HNSW__NodeDistFarther_t { private transient long swigCPtr; protected SWIGTYPE_p_std__vectorT_faiss__HNSW__NodeDistFarther_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__vectorT_faiss__HNSW__NodeDistFarther_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__vectorT_faiss__HNSW__NodeDistFarther_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__vectorT_faiss__Index_p_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__vectorT_faiss__Index_p_t { private transient long swigCPtr; protected SWIGTYPE_p_std__vectorT_faiss__Index_p_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__vectorT_faiss__Index_p_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__vectorT_faiss__Index_p_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__vectorT_faiss__InvertedLists_const_p_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__vectorT_faiss__InvertedLists_const_p_t { private transient long swigCPtr; protected SWIGTYPE_p_std__vectorT_faiss__InvertedLists_const_p_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__vectorT_faiss__InvertedLists_const_p_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__vectorT_faiss__InvertedLists_const_p_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__vectorT_faiss__OnDiskOneList_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__vectorT_faiss__OnDiskOneList_t { private transient long swigCPtr; protected SWIGTYPE_p_std__vectorT_faiss__OnDiskOneList_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__vectorT_faiss__OnDiskOneList_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__vectorT_faiss__OnDiskOneList_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__vectorT_faiss__ParameterRange_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__vectorT_faiss__ParameterRange_t { private transient long swigCPtr; protected SWIGTYPE_p_std__vectorT_faiss__ParameterRange_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__vectorT_faiss__ParameterRange_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__vectorT_faiss__ParameterRange_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__vectorT_faiss__RangeQueryResult_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__vectorT_faiss__RangeQueryResult_t { private transient long swigCPtr; protected SWIGTYPE_p_std__vectorT_faiss__RangeQueryResult_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__vectorT_faiss__RangeQueryResult_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__vectorT_faiss__RangeQueryResult_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__vectorT_faiss__RangeSearchPartialResult_p_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__vectorT_faiss__RangeSearchPartialResult_p_t { private transient long swigCPtr; protected SWIGTYPE_p_std__vectorT_faiss__RangeSearchPartialResult_p_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__vectorT_faiss__RangeSearchPartialResult_p_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__vectorT_faiss__RangeSearchPartialResult_p_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__vectorT_int64_t_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__vectorT_int64_t_t { private transient long swigCPtr; protected SWIGTYPE_p_std__vectorT_int64_t_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__vectorT_int64_t_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__vectorT_int64_t_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__vectorT_long_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__vectorT_long_t { private transient long swigCPtr; protected SWIGTYPE_p_std__vectorT_long_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__vectorT_long_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__vectorT_long_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__vectorT_omp_lock_t_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__vectorT_omp_lock_t_t { private transient long swigCPtr; protected SWIGTYPE_p_std__vectorT_omp_lock_t_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__vectorT_omp_lock_t_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__vectorT_omp_lock_t_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__vectorT_std__vectorT_int64_t_t_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__vectorT_std__vectorT_int64_t_t_t { private transient long swigCPtr; protected SWIGTYPE_p_std__vectorT_std__vectorT_int64_t_t_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__vectorT_std__vectorT_int64_t_t_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__vectorT_std__vectorT_int64_t_t_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_std__vectorT_std__vectorT_unsigned_long_t_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_std__vectorT_std__vectorT_unsigned_long_t_t { private transient long swigCPtr; protected SWIGTYPE_p_std__vectorT_std__vectorT_unsigned_long_t_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_std__vectorT_std__vectorT_unsigned_long_t_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_std__vectorT_std__vectorT_unsigned_long_t_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_uint16_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_uint16_t { private transient long swigCPtr; protected SWIGTYPE_p_uint16_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_uint16_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_uint16_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_uint32_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_uint32_t { private transient long swigCPtr; protected SWIGTYPE_p_uint32_t(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_uint32_t() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_uint32_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_unsigned_char.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_unsigned_char { private transient long swigCPtr; protected SWIGTYPE_p_unsigned_char(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_unsigned_char() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_unsigned_char obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_unsigned_long.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_unsigned_long { private transient long swigCPtr; protected SWIGTYPE_p_unsigned_long(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_unsigned_long() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_unsigned_long obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SWIGTYPE_p_void.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SWIGTYPE_p_void { private transient long swigCPtr; protected SWIGTYPE_p_void(long cPtr, @SuppressWarnings("unused") boolean futureUse) { swigCPtr = cPtr; } protected SWIGTYPE_p_void() { swigCPtr = 0; } protected static long getCPtr(SWIGTYPE_p_void obj) { return (obj == null) ? 0 : obj.swigCPtr; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SimulatedAnnealingOptimizer.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SimulatedAnnealingOptimizer extends SimulatedAnnealingParameters { private transient long swigCPtr; protected SimulatedAnnealingOptimizer(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.SimulatedAnnealingOptimizer_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(SimulatedAnnealingOptimizer obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_SimulatedAnnealingOptimizer(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setObj(PermutationObjective value) { swigfaissJNI.SimulatedAnnealingOptimizer_obj_set(swigCPtr, this, PermutationObjective.getCPtr(value), value); } public PermutationObjective getObj() { long cPtr = swigfaissJNI.SimulatedAnnealingOptimizer_obj_get(swigCPtr, this); return (cPtr == 0) ? null : new PermutationObjective(cPtr, false); } public void setN(int value) { swigfaissJNI.SimulatedAnnealingOptimizer_n_set(swigCPtr, this, value); } public int getN() { return swigfaissJNI.SimulatedAnnealingOptimizer_n_get(swigCPtr, this); } public void setLogfile(SWIGTYPE_p_FILE value) { swigfaissJNI.SimulatedAnnealingOptimizer_logfile_set(swigCPtr, this, SWIGTYPE_p_FILE.getCPtr(value)); } public SWIGTYPE_p_FILE getLogfile() { long cPtr = swigfaissJNI.SimulatedAnnealingOptimizer_logfile_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_FILE(cPtr, false); } public SimulatedAnnealingOptimizer(PermutationObjective obj, SimulatedAnnealingParameters p) { this(swigfaissJNI.new_SimulatedAnnealingOptimizer(PermutationObjective.getCPtr(obj), obj, SimulatedAnnealingParameters.getCPtr(p), p), true); } public void setRnd(SWIGTYPE_p_faiss__RandomGenerator value) { swigfaissJNI.SimulatedAnnealingOptimizer_rnd_set(swigCPtr, this, SWIGTYPE_p_faiss__RandomGenerator.getCPtr(value)); } public SWIGTYPE_p_faiss__RandomGenerator getRnd() { long cPtr = swigfaissJNI.SimulatedAnnealingOptimizer_rnd_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_faiss__RandomGenerator(cPtr, false); } public void setInit_cost(double value) { swigfaissJNI.SimulatedAnnealingOptimizer_init_cost_set(swigCPtr, this, value); } public double getInit_cost() { return swigfaissJNI.SimulatedAnnealingOptimizer_init_cost_get(swigCPtr, this); } public double optimize(SWIGTYPE_p_int perm) { return swigfaissJNI.SimulatedAnnealingOptimizer_optimize(swigCPtr, this, SWIGTYPE_p_int.getCPtr(perm)); } public double run_optimization(SWIGTYPE_p_int best_perm) { return swigfaissJNI.SimulatedAnnealingOptimizer_run_optimization(swigCPtr, this, SWIGTYPE_p_int.getCPtr(best_perm)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SimulatedAnnealingParameters.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SimulatedAnnealingParameters { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected SimulatedAnnealingParameters(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(SimulatedAnnealingParameters obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_SimulatedAnnealingParameters(swigCPtr); } swigCPtr = 0; } } public void setInit_temperature(double value) { swigfaissJNI.SimulatedAnnealingParameters_init_temperature_set(swigCPtr, this, value); } public double getInit_temperature() { return swigfaissJNI.SimulatedAnnealingParameters_init_temperature_get(swigCPtr, this); } public void setTemperature_decay(double value) { swigfaissJNI.SimulatedAnnealingParameters_temperature_decay_set(swigCPtr, this, value); } public double getTemperature_decay() { return swigfaissJNI.SimulatedAnnealingParameters_temperature_decay_get(swigCPtr, this); } public void setN_iter(int value) { swigfaissJNI.SimulatedAnnealingParameters_n_iter_set(swigCPtr, this, value); } public int getN_iter() { return swigfaissJNI.SimulatedAnnealingParameters_n_iter_get(swigCPtr, this); } public void setN_redo(int value) { swigfaissJNI.SimulatedAnnealingParameters_n_redo_set(swigCPtr, this, value); } public int getN_redo() { return swigfaissJNI.SimulatedAnnealingParameters_n_redo_get(swigCPtr, this); } public void setSeed(int value) { swigfaissJNI.SimulatedAnnealingParameters_seed_set(swigCPtr, this, value); } public int getSeed() { return swigfaissJNI.SimulatedAnnealingParameters_seed_get(swigCPtr, this); } public void setVerbose(int value) { swigfaissJNI.SimulatedAnnealingParameters_verbose_set(swigCPtr, this, value); } public int getVerbose() { return swigfaissJNI.SimulatedAnnealingParameters_verbose_get(swigCPtr, this); } public void setOnly_bit_flips(boolean value) { swigfaissJNI.SimulatedAnnealingParameters_only_bit_flips_set(swigCPtr, this, value); } public boolean getOnly_bit_flips() { return swigfaissJNI.SimulatedAnnealingParameters_only_bit_flips_get(swigCPtr, this); } public void setInit_random(boolean value) { swigfaissJNI.SimulatedAnnealingParameters_init_random_set(swigCPtr, this, value); } public boolean getInit_random() { return swigfaissJNI.SimulatedAnnealingParameters_init_random_get(swigCPtr, this); } public SimulatedAnnealingParameters() { this(swigfaissJNI.new_SimulatedAnnealingParameters(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SliceInvertedLists.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SliceInvertedLists extends ReadOnlyInvertedLists { private transient long swigCPtr; protected SliceInvertedLists(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.SliceInvertedLists_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(SliceInvertedLists obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_SliceInvertedLists(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setIl(InvertedLists value) { swigfaissJNI.SliceInvertedLists_il_set(swigCPtr, this, InvertedLists.getCPtr(value), value); } public InvertedLists getIl() { long cPtr = swigfaissJNI.SliceInvertedLists_il_get(swigCPtr, this); return (cPtr == 0) ? null : new InvertedLists(cPtr, false); } public void setI0(long value) { swigfaissJNI.SliceInvertedLists_i0_set(swigCPtr, this, value); } public long getI0() { return swigfaissJNI.SliceInvertedLists_i0_get(swigCPtr, this); } public void setI1(long value) { swigfaissJNI.SliceInvertedLists_i1_set(swigCPtr, this, value); } public long getI1() { return swigfaissJNI.SliceInvertedLists_i1_get(swigCPtr, this); } public SliceInvertedLists(InvertedLists il, long i0, long i1) { this(swigfaissJNI.new_SliceInvertedLists(InvertedLists.getCPtr(il), il, i0, i1), true); } public long list_size(long list_no) { return swigfaissJNI.SliceInvertedLists_list_size(swigCPtr, this, list_no); } public SWIGTYPE_p_unsigned_char get_codes(long list_no) { long cPtr = swigfaissJNI.SliceInvertedLists_get_codes(swigCPtr, this, list_no); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public LongVector get_ids(long list_no) { return new LongVector(swigfaissJNI.SliceInvertedLists_get_ids(swigCPtr, this, list_no), false); } public void release_codes(long list_no, SWIGTYPE_p_unsigned_char codes) { swigfaissJNI.SliceInvertedLists_release_codes(swigCPtr, this, list_no, SWIGTYPE_p_unsigned_char.getCPtr(codes)); } public void release_ids(long list_no, LongVector ids) { swigfaissJNI.SliceInvertedLists_release_ids(swigCPtr, this, list_no, SWIGTYPE_p_long_long.getCPtr(ids.data()), ids); } public long get_single_id(long list_no, long offset) { return swigfaissJNI.SliceInvertedLists_get_single_id(swigCPtr, this, list_no, offset); } public SWIGTYPE_p_unsigned_char get_single_code(long list_no, long offset) { long cPtr = swigfaissJNI.SliceInvertedLists_get_single_code(swigCPtr, this, list_no, offset); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public void prefetch_lists(LongVector list_nos, int nlist) { swigfaissJNI.SliceInvertedLists_prefetch_lists(swigCPtr, this, SWIGTYPE_p_long_long.getCPtr(list_nos.data()), list_nos, nlist); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/SlidingIndexWindow.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class SlidingIndexWindow { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected SlidingIndexWindow(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(SlidingIndexWindow obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_SlidingIndexWindow(swigCPtr); } swigCPtr = 0; } } public void setIndex(Index value) { swigfaissJNI.SlidingIndexWindow_index_set(swigCPtr, this, Index.getCPtr(value), value); } public Index getIndex() { long cPtr = swigfaissJNI.SlidingIndexWindow_index_get(swigCPtr, this); return (cPtr == 0) ? null : new Index(cPtr, false); } public void setIls(ArrayInvertedLists value) { swigfaissJNI.SlidingIndexWindow_ils_set(swigCPtr, this, ArrayInvertedLists.getCPtr(value), value); } public ArrayInvertedLists getIls() { long cPtr = swigfaissJNI.SlidingIndexWindow_ils_get(swigCPtr, this); return (cPtr == 0) ? null : new ArrayInvertedLists(cPtr, false); } public void setN_slice(int value) { swigfaissJNI.SlidingIndexWindow_n_slice_set(swigCPtr, this, value); } public int getN_slice() { return swigfaissJNI.SlidingIndexWindow_n_slice_get(swigCPtr, this); } public void setNlist(long value) { swigfaissJNI.SlidingIndexWindow_nlist_set(swigCPtr, this, value); } public long getNlist() { return swigfaissJNI.SlidingIndexWindow_nlist_get(swigCPtr, this); } public void setSizes(SWIGTYPE_p_std__vectorT_std__vectorT_unsigned_long_t_t value) { swigfaissJNI.SlidingIndexWindow_sizes_set(swigCPtr, this, SWIGTYPE_p_std__vectorT_std__vectorT_unsigned_long_t_t.getCPtr(value)); } public SWIGTYPE_p_std__vectorT_std__vectorT_unsigned_long_t_t getSizes() { long cPtr = swigfaissJNI.SlidingIndexWindow_sizes_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_std__vectorT_unsigned_long_t_t(cPtr, false); } public SlidingIndexWindow(Index index) { this(swigfaissJNI.new_SlidingIndexWindow(Index.getCPtr(index), index), true); } public void step(Index sub_index, boolean remove_oldest) { swigfaissJNI.SlidingIndexWindow_step(swigCPtr, this, Index.getCPtr(sub_index), sub_index, remove_oldest); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/StopWordsInvertedLists.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class StopWordsInvertedLists extends ReadOnlyInvertedLists { private transient long swigCPtr; protected StopWordsInvertedLists(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.StopWordsInvertedLists_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(StopWordsInvertedLists obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_StopWordsInvertedLists(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setIl0(InvertedLists value) { swigfaissJNI.StopWordsInvertedLists_il0_set(swigCPtr, this, InvertedLists.getCPtr(value), value); } public InvertedLists getIl0() { long cPtr = swigfaissJNI.StopWordsInvertedLists_il0_get(swigCPtr, this); return (cPtr == 0) ? null : new InvertedLists(cPtr, false); } public void setMaxsize(long value) { swigfaissJNI.StopWordsInvertedLists_maxsize_set(swigCPtr, this, value); } public long getMaxsize() { return swigfaissJNI.StopWordsInvertedLists_maxsize_get(swigCPtr, this); } public StopWordsInvertedLists(InvertedLists il, long maxsize) { this(swigfaissJNI.new_StopWordsInvertedLists(InvertedLists.getCPtr(il), il, maxsize), true); } public long list_size(long list_no) { return swigfaissJNI.StopWordsInvertedLists_list_size(swigCPtr, this, list_no); } public SWIGTYPE_p_unsigned_char get_codes(long list_no) { long cPtr = swigfaissJNI.StopWordsInvertedLists_get_codes(swigCPtr, this, list_no); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public LongVector get_ids(long list_no) { return new LongVector(swigfaissJNI.StopWordsInvertedLists_get_ids(swigCPtr, this, list_no), false); } public void release_codes(long list_no, SWIGTYPE_p_unsigned_char codes) { swigfaissJNI.StopWordsInvertedLists_release_codes(swigCPtr, this, list_no, SWIGTYPE_p_unsigned_char.getCPtr(codes)); } public void release_ids(long list_no, LongVector ids) { swigfaissJNI.StopWordsInvertedLists_release_ids(swigCPtr, this, list_no, SWIGTYPE_p_long_long.getCPtr(ids.data()), ids); } public long get_single_id(long list_no, long offset) { return swigfaissJNI.StopWordsInvertedLists_get_single_id(swigCPtr, this, list_no, offset); } public SWIGTYPE_p_unsigned_char get_single_code(long list_no, long offset) { long cPtr = swigfaissJNI.StopWordsInvertedLists_get_single_code(swigCPtr, this, list_no, offset); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public void prefetch_lists(LongVector list_nos, int nlist) { swigfaissJNI.StopWordsInvertedLists_prefetch_lists(swigCPtr, this, SWIGTYPE_p_long_long.getCPtr(list_nos.data()), list_nos, nlist); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/Uint64Vector.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class Uint64Vector { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected Uint64Vector(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(Uint64Vector obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_Uint64Vector(swigCPtr); } swigCPtr = 0; } } public Uint64Vector() { this(swigfaissJNI.new_Uint64Vector(), true); } public void push_back(long arg0) { swigfaissJNI.Uint64Vector_push_back(swigCPtr, this, arg0); } public void clear() { swigfaissJNI.Uint64Vector_clear(swigCPtr, this); } public SWIGTYPE_p_unsigned_long data() { long cPtr = swigfaissJNI.Uint64Vector_data(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_long(cPtr, false); } public long size() { return swigfaissJNI.Uint64Vector_size(swigCPtr, this); } public long at(long n) { return swigfaissJNI.Uint64Vector_at(swigCPtr, this, n); } public void resize(long n) { swigfaissJNI.Uint64Vector_resize(swigCPtr, this, n); } public void reserve(long n) { swigfaissJNI.Uint64Vector_reserve(swigCPtr, this, n); } public void swap(Uint64Vector other) { swigfaissJNI.Uint64Vector_swap(swigCPtr, this, Uint64Vector.getCPtr(other), other); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/VStackInvertedLists.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class VStackInvertedLists extends ReadOnlyInvertedLists { private transient long swigCPtr; protected VStackInvertedLists(long cPtr, boolean cMemoryOwn) { super(swigfaissJNI.VStackInvertedLists_SWIGUpcast(cPtr), cMemoryOwn); swigCPtr = cPtr; } protected static long getCPtr(VStackInvertedLists obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_VStackInvertedLists(swigCPtr); } swigCPtr = 0; } super.delete(); } public void setIls(SWIGTYPE_p_std__vectorT_faiss__InvertedLists_const_p_t value) { swigfaissJNI.VStackInvertedLists_ils_set(swigCPtr, this, SWIGTYPE_p_std__vectorT_faiss__InvertedLists_const_p_t.getCPtr(value)); } public SWIGTYPE_p_std__vectorT_faiss__InvertedLists_const_p_t getIls() { long cPtr = swigfaissJNI.VStackInvertedLists_ils_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_faiss__InvertedLists_const_p_t(cPtr, false); } public void setCumsz(SWIGTYPE_p_std__vectorT_int64_t_t value) { swigfaissJNI.VStackInvertedLists_cumsz_set(swigCPtr, this, SWIGTYPE_p_std__vectorT_int64_t_t.getCPtr(value)); } public SWIGTYPE_p_std__vectorT_int64_t_t getCumsz() { long cPtr = swigfaissJNI.VStackInvertedLists_cumsz_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_std__vectorT_int64_t_t(cPtr, false); } public VStackInvertedLists(int nil, SWIGTYPE_p_p_faiss__InvertedLists ils) { this(swigfaissJNI.new_VStackInvertedLists(nil, SWIGTYPE_p_p_faiss__InvertedLists.getCPtr(ils)), true); } public long list_size(long list_no) { return swigfaissJNI.VStackInvertedLists_list_size(swigCPtr, this, list_no); } public SWIGTYPE_p_unsigned_char get_codes(long list_no) { long cPtr = swigfaissJNI.VStackInvertedLists_get_codes(swigCPtr, this, list_no); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public LongVector get_ids(long list_no) { return new LongVector(swigfaissJNI.VStackInvertedLists_get_ids(swigCPtr, this, list_no), false); } public void release_codes(long list_no, SWIGTYPE_p_unsigned_char codes) { swigfaissJNI.VStackInvertedLists_release_codes(swigCPtr, this, list_no, SWIGTYPE_p_unsigned_char.getCPtr(codes)); } public void release_ids(long list_no, LongVector ids) { swigfaissJNI.VStackInvertedLists_release_ids(swigCPtr, this, list_no, SWIGTYPE_p_long_long.getCPtr(ids.data()), ids); } public long get_single_id(long list_no, long offset) { return swigfaissJNI.VStackInvertedLists_get_single_id(swigCPtr, this, list_no, offset); } public SWIGTYPE_p_unsigned_char get_single_code(long list_no, long offset) { long cPtr = swigfaissJNI.VStackInvertedLists_get_single_code(swigCPtr, this, list_no, offset); return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); } public void prefetch_lists(LongVector list_nos, int nlist) { swigfaissJNI.VStackInvertedLists_prefetch_lists(swigCPtr, this, SWIGTYPE_p_long_long.getCPtr(list_nos.data()), list_nos, nlist); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/VectorTransform.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class VectorTransform { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected VectorTransform(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(VectorTransform obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_VectorTransform(swigCPtr); } swigCPtr = 0; } } public void setD_in(int value) { swigfaissJNI.VectorTransform_d_in_set(swigCPtr, this, value); } public int getD_in() { return swigfaissJNI.VectorTransform_d_in_get(swigCPtr, this); } public void setD_out(int value) { swigfaissJNI.VectorTransform_d_out_set(swigCPtr, this, value); } public int getD_out() { return swigfaissJNI.VectorTransform_d_out_get(swigCPtr, this); } public void setIs_trained(boolean value) { swigfaissJNI.VectorTransform_is_trained_set(swigCPtr, this, value); } public boolean getIs_trained() { return swigfaissJNI.VectorTransform_is_trained_get(swigCPtr, this); } public void train(long n, SWIGTYPE_p_float x) { swigfaissJNI.VectorTransform_train(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); } public SWIGTYPE_p_float apply(long n, SWIGTYPE_p_float x) { long cPtr = swigfaissJNI.VectorTransform_apply(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x)); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } public void apply_noalloc(long n, SWIGTYPE_p_float x, SWIGTYPE_p_float xt) { swigfaissJNI.VectorTransform_apply_noalloc(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_float.getCPtr(xt)); } public void reverse_transform(long n, SWIGTYPE_p_float xt, SWIGTYPE_p_float x) { swigfaissJNI.VectorTransform_reverse_transform(swigCPtr, this, n, SWIGTYPE_p_float.getCPtr(xt), SWIGTYPE_p_float.getCPtr(x)); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/VectorTransformVector.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class VectorTransformVector { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected VectorTransformVector(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(VectorTransformVector obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_VectorTransformVector(swigCPtr); } swigCPtr = 0; } } public VectorTransformVector() { this(swigfaissJNI.new_VectorTransformVector(), true); } public void push_back(VectorTransform arg0) { swigfaissJNI.VectorTransformVector_push_back(swigCPtr, this, VectorTransform.getCPtr(arg0), arg0); } public void clear() { swigfaissJNI.VectorTransformVector_clear(swigCPtr, this); } public SWIGTYPE_p_p_faiss__VectorTransform data() { long cPtr = swigfaissJNI.VectorTransformVector_data(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_p_faiss__VectorTransform(cPtr, false); } public long size() { return swigfaissJNI.VectorTransformVector_size(swigCPtr, this); } public VectorTransform at(long n) { long cPtr = swigfaissJNI.VectorTransformVector_at(swigCPtr, this, n); return (cPtr == 0) ? null : new VectorTransform(cPtr, false); } public void resize(long n) { swigfaissJNI.VectorTransformVector_resize(swigCPtr, this, n); } public void reserve(long n) { swigfaissJNI.VectorTransformVector_reserve(swigCPtr, this, n); } public void swap(VectorTransformVector other) { swigfaissJNI.VectorTransformVector_swap(swigCPtr, this, VectorTransformVector.getCPtr(other), other); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/VisitedTable.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class VisitedTable { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected VisitedTable(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(VisitedTable obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_VisitedTable(swigCPtr); } swigCPtr = 0; } } public void setVisited(ByteVector value) { swigfaissJNI.VisitedTable_visited_set(swigCPtr, this, ByteVector.getCPtr(value), value); } public ByteVector getVisited() { long cPtr = swigfaissJNI.VisitedTable_visited_get(swigCPtr, this); return (cPtr == 0) ? null : new ByteVector(cPtr, false); } public void setVisno(int value) { swigfaissJNI.VisitedTable_visno_set(swigCPtr, this, value); } public int getVisno() { return swigfaissJNI.VisitedTable_visno_get(swigCPtr, this); } public VisitedTable(int size) { this(swigfaissJNI.new_VisitedTable(size), true); } public void set(int no) { swigfaissJNI.VisitedTable_set(swigCPtr, this, no); } public boolean get(int no) { return swigfaissJNI.VisitedTable_get(swigCPtr, this, no); } public void advance() { swigfaissJNI.VisitedTable_advance(swigCPtr, this); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/doubleArray.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class doubleArray { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected doubleArray(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(doubleArray obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_doubleArray(swigCPtr); } swigCPtr = 0; } } public doubleArray(int nelements) { this(swigfaissJNI.new_doubleArray(nelements), true); } public double getitem(int index) { return swigfaissJNI.doubleArray_getitem(swigCPtr, this, index); } public void setitem(int index, double value) { swigfaissJNI.doubleArray_setitem(swigCPtr, this, index, value); } public SWIGTYPE_p_double cast() { long cPtr = swigfaissJNI.doubleArray_cast(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_double(cPtr, false); } public static doubleArray frompointer(SWIGTYPE_p_double t) { long cPtr = swigfaissJNI.doubleArray_frompointer(SWIGTYPE_p_double.getCPtr(t)); return (cPtr == 0) ? null : new doubleArray(cPtr, false); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/floatArray.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class floatArray { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected floatArray(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(floatArray obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_floatArray(swigCPtr); } swigCPtr = 0; } } public floatArray(int nelements) { this(swigfaissJNI.new_floatArray(nelements), true); } public float getitem(int index) { return swigfaissJNI.floatArray_getitem(swigCPtr, this, index); } public void setitem(int index, float value) { swigfaissJNI.floatArray_setitem(swigCPtr, this, index, value); } public SWIGTYPE_p_float cast() { long cPtr = swigfaissJNI.floatArray_cast(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } public static floatArray frompointer(SWIGTYPE_p_float t) { long cPtr = swigfaissJNI.floatArray_frompointer(SWIGTYPE_p_float.getCPtr(t)); return (cPtr == 0) ? null : new floatArray(cPtr, false); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/float_maxheap_array_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class float_maxheap_array_t { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected float_maxheap_array_t(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(float_maxheap_array_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_float_maxheap_array_t(swigCPtr); } swigCPtr = 0; } } public void setNh(long value) { swigfaissJNI.float_maxheap_array_t_nh_set(swigCPtr, this, value); } public long getNh() { return swigfaissJNI.float_maxheap_array_t_nh_get(swigCPtr, this); } public void setK(long value) { swigfaissJNI.float_maxheap_array_t_k_set(swigCPtr, this, value); } public long getK() { return swigfaissJNI.float_maxheap_array_t_k_get(swigCPtr, this); } public void setIds(LongVector value) { swigfaissJNI.float_maxheap_array_t_ids_set(swigCPtr, this, SWIGTYPE_p_long_long.getCPtr(value.data()), value); } public LongVector getIds() { return new LongVector(swigfaissJNI.float_maxheap_array_t_ids_get(swigCPtr, this), false); } public void setVal(SWIGTYPE_p_float value) { swigfaissJNI.float_maxheap_array_t_val_set(swigCPtr, this, SWIGTYPE_p_float.getCPtr(value)); } public SWIGTYPE_p_float getVal() { long cPtr = swigfaissJNI.float_maxheap_array_t_val_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } public SWIGTYPE_p_float get_val(long key) { long cPtr = swigfaissJNI.float_maxheap_array_t_get_val(swigCPtr, this, key); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } public LongVector get_ids(long key) { return new LongVector(swigfaissJNI.float_maxheap_array_t_get_ids(swigCPtr, this, key), false); } public void heapify() { swigfaissJNI.float_maxheap_array_t_heapify(swigCPtr, this); } public void addn(long nj, SWIGTYPE_p_float vin, long j0, long i0, long ni) { swigfaissJNI.float_maxheap_array_t_addn__SWIG_0(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin), j0, i0, ni); } public void addn(long nj, SWIGTYPE_p_float vin, long j0, long i0) { swigfaissJNI.float_maxheap_array_t_addn__SWIG_1(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin), j0, i0); } public void addn(long nj, SWIGTYPE_p_float vin, long j0) { swigfaissJNI.float_maxheap_array_t_addn__SWIG_2(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin), j0); } public void addn(long nj, SWIGTYPE_p_float vin) { swigfaissJNI.float_maxheap_array_t_addn__SWIG_3(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin)); } public void addn_with_ids(long nj, SWIGTYPE_p_float vin, LongVector id_in, long id_stride, long i0, long ni) { swigfaissJNI.float_maxheap_array_t_addn_with_ids__SWIG_0(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin), SWIGTYPE_p_long_long.getCPtr(id_in.data()), id_in, id_stride, i0, ni); } public void addn_with_ids(long nj, SWIGTYPE_p_float vin, LongVector id_in, long id_stride, long i0) { swigfaissJNI.float_maxheap_array_t_addn_with_ids__SWIG_1(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin), SWIGTYPE_p_long_long.getCPtr(id_in.data()), id_in, id_stride, i0); } public void addn_with_ids(long nj, SWIGTYPE_p_float vin, LongVector id_in, long id_stride) { swigfaissJNI.float_maxheap_array_t_addn_with_ids__SWIG_2(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin), SWIGTYPE_p_long_long.getCPtr(id_in.data()), id_in, id_stride); } public void addn_with_ids(long nj, SWIGTYPE_p_float vin, LongVector id_in) { swigfaissJNI.float_maxheap_array_t_addn_with_ids__SWIG_3(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin), SWIGTYPE_p_long_long.getCPtr(id_in.data()), id_in); } public void addn_with_ids(long nj, SWIGTYPE_p_float vin) { swigfaissJNI.float_maxheap_array_t_addn_with_ids__SWIG_4(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin)); } public void reorder() { swigfaissJNI.float_maxheap_array_t_reorder(swigCPtr, this); } public void per_line_extrema(SWIGTYPE_p_float vals_out, LongVector idx_out) { swigfaissJNI.float_maxheap_array_t_per_line_extrema(swigCPtr, this, SWIGTYPE_p_float.getCPtr(vals_out), SWIGTYPE_p_long_long.getCPtr(idx_out.data()), idx_out); } public float_maxheap_array_t() { this(swigfaissJNI.new_float_maxheap_array_t(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/float_minheap_array_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class float_minheap_array_t { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected float_minheap_array_t(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(float_minheap_array_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_float_minheap_array_t(swigCPtr); } swigCPtr = 0; } } public void setNh(long value) { swigfaissJNI.float_minheap_array_t_nh_set(swigCPtr, this, value); } public long getNh() { return swigfaissJNI.float_minheap_array_t_nh_get(swigCPtr, this); } public void setK(long value) { swigfaissJNI.float_minheap_array_t_k_set(swigCPtr, this, value); } public long getK() { return swigfaissJNI.float_minheap_array_t_k_get(swigCPtr, this); } public void setIds(LongVector value) { swigfaissJNI.float_minheap_array_t_ids_set(swigCPtr, this, SWIGTYPE_p_long_long.getCPtr(value.data()), value); } public LongVector getIds() { return new LongVector(swigfaissJNI.float_minheap_array_t_ids_get(swigCPtr, this), false); } public void setVal(SWIGTYPE_p_float value) { swigfaissJNI.float_minheap_array_t_val_set(swigCPtr, this, SWIGTYPE_p_float.getCPtr(value)); } public SWIGTYPE_p_float getVal() { long cPtr = swigfaissJNI.float_minheap_array_t_val_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } public SWIGTYPE_p_float get_val(long key) { long cPtr = swigfaissJNI.float_minheap_array_t_get_val(swigCPtr, this, key); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } public LongVector get_ids(long key) { return new LongVector(swigfaissJNI.float_minheap_array_t_get_ids(swigCPtr, this, key), false); } public void heapify() { swigfaissJNI.float_minheap_array_t_heapify(swigCPtr, this); } public void addn(long nj, SWIGTYPE_p_float vin, long j0, long i0, long ni) { swigfaissJNI.float_minheap_array_t_addn__SWIG_0(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin), j0, i0, ni); } public void addn(long nj, SWIGTYPE_p_float vin, long j0, long i0) { swigfaissJNI.float_minheap_array_t_addn__SWIG_1(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin), j0, i0); } public void addn(long nj, SWIGTYPE_p_float vin, long j0) { swigfaissJNI.float_minheap_array_t_addn__SWIG_2(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin), j0); } public void addn(long nj, SWIGTYPE_p_float vin) { swigfaissJNI.float_minheap_array_t_addn__SWIG_3(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin)); } public void addn_with_ids(long nj, SWIGTYPE_p_float vin, LongVector id_in, long id_stride, long i0, long ni) { swigfaissJNI.float_minheap_array_t_addn_with_ids__SWIG_0(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin), SWIGTYPE_p_long_long.getCPtr(id_in.data()), id_in, id_stride, i0, ni); } public void addn_with_ids(long nj, SWIGTYPE_p_float vin, LongVector id_in, long id_stride, long i0) { swigfaissJNI.float_minheap_array_t_addn_with_ids__SWIG_1(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin), SWIGTYPE_p_long_long.getCPtr(id_in.data()), id_in, id_stride, i0); } public void addn_with_ids(long nj, SWIGTYPE_p_float vin, LongVector id_in, long id_stride) { swigfaissJNI.float_minheap_array_t_addn_with_ids__SWIG_2(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin), SWIGTYPE_p_long_long.getCPtr(id_in.data()), id_in, id_stride); } public void addn_with_ids(long nj, SWIGTYPE_p_float vin, LongVector id_in) { swigfaissJNI.float_minheap_array_t_addn_with_ids__SWIG_3(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin), SWIGTYPE_p_long_long.getCPtr(id_in.data()), id_in); } public void addn_with_ids(long nj, SWIGTYPE_p_float vin) { swigfaissJNI.float_minheap_array_t_addn_with_ids__SWIG_4(swigCPtr, this, nj, SWIGTYPE_p_float.getCPtr(vin)); } public void reorder() { swigfaissJNI.float_minheap_array_t_reorder(swigCPtr, this); } public void per_line_extrema(SWIGTYPE_p_float vals_out, LongVector idx_out) { swigfaissJNI.float_minheap_array_t_per_line_extrema(swigCPtr, this, SWIGTYPE_p_float.getCPtr(vals_out), SWIGTYPE_p_long_long.getCPtr(idx_out.data()), idx_out); } public float_minheap_array_t() { this(swigfaissJNI.new_float_minheap_array_t(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/intArray.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class intArray { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected intArray(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(intArray obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_intArray(swigCPtr); } swigCPtr = 0; } } public intArray(int nelements) { this(swigfaissJNI.new_intArray(nelements), true); } public int getitem(int index) { return swigfaissJNI.intArray_getitem(swigCPtr, this, index); } public void setitem(int index, int value) { swigfaissJNI.intArray_setitem(swigCPtr, this, index, value); } public SWIGTYPE_p_int cast() { long cPtr = swigfaissJNI.intArray_cast(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_int(cPtr, false); } public static intArray frompointer(SWIGTYPE_p_int t) { long cPtr = swigfaissJNI.intArray_frompointer(SWIGTYPE_p_int.getCPtr(t)); return (cPtr == 0) ? null : new intArray(cPtr, false); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/int_maxheap_array_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class int_maxheap_array_t { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected int_maxheap_array_t(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(int_maxheap_array_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_int_maxheap_array_t(swigCPtr); } swigCPtr = 0; } } public void setNh(long value) { swigfaissJNI.int_maxheap_array_t_nh_set(swigCPtr, this, value); } public long getNh() { return swigfaissJNI.int_maxheap_array_t_nh_get(swigCPtr, this); } public void setK(long value) { swigfaissJNI.int_maxheap_array_t_k_set(swigCPtr, this, value); } public long getK() { return swigfaissJNI.int_maxheap_array_t_k_get(swigCPtr, this); } public void setIds(LongVector value) { swigfaissJNI.int_maxheap_array_t_ids_set(swigCPtr, this, SWIGTYPE_p_long_long.getCPtr(value.data()), value); } public LongVector getIds() { return new LongVector(swigfaissJNI.int_maxheap_array_t_ids_get(swigCPtr, this), false); } public void setVal(SWIGTYPE_p_int value) { swigfaissJNI.int_maxheap_array_t_val_set(swigCPtr, this, SWIGTYPE_p_int.getCPtr(value)); } public SWIGTYPE_p_int getVal() { long cPtr = swigfaissJNI.int_maxheap_array_t_val_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_int(cPtr, false); } public SWIGTYPE_p_int get_val(long key) { long cPtr = swigfaissJNI.int_maxheap_array_t_get_val(swigCPtr, this, key); return (cPtr == 0) ? null : new SWIGTYPE_p_int(cPtr, false); } public LongVector get_ids(long key) { return new LongVector(swigfaissJNI.int_maxheap_array_t_get_ids(swigCPtr, this, key), false); } public void heapify() { swigfaissJNI.int_maxheap_array_t_heapify(swigCPtr, this); } public void addn(long nj, SWIGTYPE_p_int vin, long j0, long i0, long ni) { swigfaissJNI.int_maxheap_array_t_addn__SWIG_0(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin), j0, i0, ni); } public void addn(long nj, SWIGTYPE_p_int vin, long j0, long i0) { swigfaissJNI.int_maxheap_array_t_addn__SWIG_1(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin), j0, i0); } public void addn(long nj, SWIGTYPE_p_int vin, long j0) { swigfaissJNI.int_maxheap_array_t_addn__SWIG_2(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin), j0); } public void addn(long nj, SWIGTYPE_p_int vin) { swigfaissJNI.int_maxheap_array_t_addn__SWIG_3(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin)); } public void addn_with_ids(long nj, SWIGTYPE_p_int vin, LongVector id_in, long id_stride, long i0, long ni) { swigfaissJNI.int_maxheap_array_t_addn_with_ids__SWIG_0(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin), SWIGTYPE_p_long_long.getCPtr(id_in.data()), id_in, id_stride, i0, ni); } public void addn_with_ids(long nj, SWIGTYPE_p_int vin, LongVector id_in, long id_stride, long i0) { swigfaissJNI.int_maxheap_array_t_addn_with_ids__SWIG_1(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin), SWIGTYPE_p_long_long.getCPtr(id_in.data()), id_in, id_stride, i0); } public void addn_with_ids(long nj, SWIGTYPE_p_int vin, LongVector id_in, long id_stride) { swigfaissJNI.int_maxheap_array_t_addn_with_ids__SWIG_2(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin), SWIGTYPE_p_long_long.getCPtr(id_in.data()), id_in, id_stride); } public void addn_with_ids(long nj, SWIGTYPE_p_int vin, LongVector id_in) { swigfaissJNI.int_maxheap_array_t_addn_with_ids__SWIG_3(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin), SWIGTYPE_p_long_long.getCPtr(id_in.data()), id_in); } public void addn_with_ids(long nj, SWIGTYPE_p_int vin) { swigfaissJNI.int_maxheap_array_t_addn_with_ids__SWIG_4(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin)); } public void reorder() { swigfaissJNI.int_maxheap_array_t_reorder(swigCPtr, this); } public void per_line_extrema(SWIGTYPE_p_int vals_out, LongVector idx_out) { swigfaissJNI.int_maxheap_array_t_per_line_extrema(swigCPtr, this, SWIGTYPE_p_int.getCPtr(vals_out), SWIGTYPE_p_long_long.getCPtr(idx_out.data()), idx_out); } public int_maxheap_array_t() { this(swigfaissJNI.new_int_maxheap_array_t(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/int_minheap_array_t.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class int_minheap_array_t { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected int_minheap_array_t(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(int_minheap_array_t obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_int_minheap_array_t(swigCPtr); } swigCPtr = 0; } } public void setNh(long value) { swigfaissJNI.int_minheap_array_t_nh_set(swigCPtr, this, value); } public long getNh() { return swigfaissJNI.int_minheap_array_t_nh_get(swigCPtr, this); } public void setK(long value) { swigfaissJNI.int_minheap_array_t_k_set(swigCPtr, this, value); } public long getK() { return swigfaissJNI.int_minheap_array_t_k_get(swigCPtr, this); } public void setIds(LongVector value) { swigfaissJNI.int_minheap_array_t_ids_set(swigCPtr, this, SWIGTYPE_p_long_long.getCPtr(value.data()), value); } public LongVector getIds() { return new LongVector(swigfaissJNI.int_minheap_array_t_ids_get(swigCPtr, this), false); } public void setVal(SWIGTYPE_p_int value) { swigfaissJNI.int_minheap_array_t_val_set(swigCPtr, this, SWIGTYPE_p_int.getCPtr(value)); } public SWIGTYPE_p_int getVal() { long cPtr = swigfaissJNI.int_minheap_array_t_val_get(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_int(cPtr, false); } public SWIGTYPE_p_int get_val(long key) { long cPtr = swigfaissJNI.int_minheap_array_t_get_val(swigCPtr, this, key); return (cPtr == 0) ? null : new SWIGTYPE_p_int(cPtr, false); } public LongVector get_ids(long key) { return new LongVector(swigfaissJNI.int_minheap_array_t_get_ids(swigCPtr, this, key), false); } public void heapify() { swigfaissJNI.int_minheap_array_t_heapify(swigCPtr, this); } public void addn(long nj, SWIGTYPE_p_int vin, long j0, long i0, long ni) { swigfaissJNI.int_minheap_array_t_addn__SWIG_0(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin), j0, i0, ni); } public void addn(long nj, SWIGTYPE_p_int vin, long j0, long i0) { swigfaissJNI.int_minheap_array_t_addn__SWIG_1(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin), j0, i0); } public void addn(long nj, SWIGTYPE_p_int vin, long j0) { swigfaissJNI.int_minheap_array_t_addn__SWIG_2(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin), j0); } public void addn(long nj, SWIGTYPE_p_int vin) { swigfaissJNI.int_minheap_array_t_addn__SWIG_3(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin)); } public void addn_with_ids(long nj, SWIGTYPE_p_int vin, LongVector id_in, long id_stride, long i0, long ni) { swigfaissJNI.int_minheap_array_t_addn_with_ids__SWIG_0(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin), SWIGTYPE_p_long_long.getCPtr(id_in.data()), id_in, id_stride, i0, ni); } public void addn_with_ids(long nj, SWIGTYPE_p_int vin, LongVector id_in, long id_stride, long i0) { swigfaissJNI.int_minheap_array_t_addn_with_ids__SWIG_1(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin), SWIGTYPE_p_long_long.getCPtr(id_in.data()), id_in, id_stride, i0); } public void addn_with_ids(long nj, SWIGTYPE_p_int vin, LongVector id_in, long id_stride) { swigfaissJNI.int_minheap_array_t_addn_with_ids__SWIG_2(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin), SWIGTYPE_p_long_long.getCPtr(id_in.data()), id_in, id_stride); } public void addn_with_ids(long nj, SWIGTYPE_p_int vin, LongVector id_in) { swigfaissJNI.int_minheap_array_t_addn_with_ids__SWIG_3(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin), SWIGTYPE_p_long_long.getCPtr(id_in.data()), id_in); } public void addn_with_ids(long nj, SWIGTYPE_p_int vin) { swigfaissJNI.int_minheap_array_t_addn_with_ids__SWIG_4(swigCPtr, this, nj, SWIGTYPE_p_int.getCPtr(vin)); } public void reorder() { swigfaissJNI.int_minheap_array_t_reorder(swigCPtr, this); } public void per_line_extrema(SWIGTYPE_p_int vals_out, LongVector idx_out) { swigfaissJNI.int_minheap_array_t_per_line_extrema(swigCPtr, this, SWIGTYPE_p_int.getCPtr(vals_out), SWIGTYPE_p_long_long.getCPtr(idx_out.data()), idx_out); } public int_minheap_array_t() { this(swigfaissJNI.new_int_minheap_array_t(), true); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/longArray.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class longArray { private transient long swigCPtr; protected transient boolean swigCMemOwn; protected longArray(long cPtr, boolean cMemoryOwn) { swigCMemOwn = cMemoryOwn; swigCPtr = cPtr; } protected static long getCPtr(longArray obj) { return (obj == null) ? 0 : obj.swigCPtr; } @SuppressWarnings("deprecation") protected void finalize() { delete(); } public synchronized void delete() { if (swigCPtr != 0) { if (swigCMemOwn) { swigCMemOwn = false; swigfaissJNI.delete_longArray(swigCPtr); } swigCPtr = 0; } } public longArray(int nelements) { this(swigfaissJNI.new_longArray(nelements), true); } public long getitem(int index) { return swigfaissJNI.longArray_getitem(swigCPtr, this, index); } public void setitem(int index, long value) { swigfaissJNI.longArray_setitem(swigCPtr, this, index, value); } public SWIGTYPE_p_long_long cast() { long cPtr = swigfaissJNI.longArray_cast(swigCPtr, this); return (cPtr == 0) ? null : new SWIGTYPE_p_long_long(cPtr, false); } public static longArray frompointer(SWIGTYPE_p_long_long t) { long cPtr = swigfaissJNI.longArray_frompointer(SWIGTYPE_p_long_long.getCPtr(t)); return (cPtr == 0) ? null : new longArray(cPtr, false); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/resources/.gitignore ================================================ *.so *.so.0 *.so.1 *.so.3 *.so.5 *.so.6 *.dylib ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/resources/.gitkeep ================================================ ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/resources/BUILD ================================================ resources( name = "resources", sources = [ "*.dylib", "*.so", "*.so.0", "*.so.1", "*.so.3", "*.so.5", "*.so.6", ], tags = [ "bazel-compatible", "bazel-only", "visibility://visibility:private", ], ) ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/swigfaiss.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public class swigfaiss implements swigfaissConstants { public static void bitvec_print(SWIGTYPE_p_unsigned_char b, long d) { swigfaissJNI.bitvec_print(SWIGTYPE_p_unsigned_char.getCPtr(b), d); } public static void fvecs2bitvecs(SWIGTYPE_p_float x, SWIGTYPE_p_unsigned_char b, long d, long n) { swigfaissJNI.fvecs2bitvecs(SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_unsigned_char.getCPtr(b), d, n); } public static void bitvecs2fvecs(SWIGTYPE_p_unsigned_char b, SWIGTYPE_p_float x, long d, long n) { swigfaissJNI.bitvecs2fvecs(SWIGTYPE_p_unsigned_char.getCPtr(b), SWIGTYPE_p_float.getCPtr(x), d, n); } public static void fvec2bitvec(SWIGTYPE_p_float x, SWIGTYPE_p_unsigned_char b, long d) { swigfaissJNI.fvec2bitvec(SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_unsigned_char.getCPtr(b), d); } public static void bitvec_shuffle(long n, long da, long db, SWIGTYPE_p_int order, SWIGTYPE_p_unsigned_char a, SWIGTYPE_p_unsigned_char b) { swigfaissJNI.bitvec_shuffle(n, da, db, SWIGTYPE_p_int.getCPtr(order), SWIGTYPE_p_unsigned_char.getCPtr(a), SWIGTYPE_p_unsigned_char.getCPtr(b)); } public static void setHamming_batch_size(long value) { swigfaissJNI.hamming_batch_size_set(value); } public static long getHamming_batch_size() { return swigfaissJNI.hamming_batch_size_get(); } public static int popcount64(long x) { return swigfaissJNI.popcount64(x); } public static void hammings(SWIGTYPE_p_unsigned_char a, SWIGTYPE_p_unsigned_char b, long na, long nb, long nbytespercode, SWIGTYPE_p_int dis) { swigfaissJNI.hammings(SWIGTYPE_p_unsigned_char.getCPtr(a), SWIGTYPE_p_unsigned_char.getCPtr(b), na, nb, nbytespercode, SWIGTYPE_p_int.getCPtr(dis)); } public static void hammings_knn_hc(SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_int_int64_t_t_t ha, SWIGTYPE_p_unsigned_char a, SWIGTYPE_p_unsigned_char b, long nb, long ncodes, int ordered) { swigfaissJNI.hammings_knn_hc(SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_int_int64_t_t_t.getCPtr(ha), SWIGTYPE_p_unsigned_char.getCPtr(a), SWIGTYPE_p_unsigned_char.getCPtr(b), nb, ncodes, ordered); } public static void hammings_knn(SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_int_int64_t_t_t ha, SWIGTYPE_p_unsigned_char a, SWIGTYPE_p_unsigned_char b, long nb, long ncodes, int ordered) { swigfaissJNI.hammings_knn(SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_int_int64_t_t_t.getCPtr(ha), SWIGTYPE_p_unsigned_char.getCPtr(a), SWIGTYPE_p_unsigned_char.getCPtr(b), nb, ncodes, ordered); } public static void hammings_knn_mc(SWIGTYPE_p_unsigned_char a, SWIGTYPE_p_unsigned_char b, long na, long nb, long k, long ncodes, SWIGTYPE_p_int distances, LongVector labels) { swigfaissJNI.hammings_knn_mc(SWIGTYPE_p_unsigned_char.getCPtr(a), SWIGTYPE_p_unsigned_char.getCPtr(b), na, nb, k, ncodes, SWIGTYPE_p_int.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels); } public static void hamming_range_search(SWIGTYPE_p_unsigned_char a, SWIGTYPE_p_unsigned_char b, long na, long nb, int radius, long ncodes, RangeSearchResult result) { swigfaissJNI.hamming_range_search(SWIGTYPE_p_unsigned_char.getCPtr(a), SWIGTYPE_p_unsigned_char.getCPtr(b), na, nb, radius, ncodes, RangeSearchResult.getCPtr(result), result); } public static void hamming_count_thres(SWIGTYPE_p_unsigned_char bs1, SWIGTYPE_p_unsigned_char bs2, long n1, long n2, int ht, long ncodes, SWIGTYPE_p_unsigned_long nptr) { swigfaissJNI.hamming_count_thres(SWIGTYPE_p_unsigned_char.getCPtr(bs1), SWIGTYPE_p_unsigned_char.getCPtr(bs2), n1, n2, ht, ncodes, SWIGTYPE_p_unsigned_long.getCPtr(nptr)); } public static long match_hamming_thres(SWIGTYPE_p_unsigned_char bs1, SWIGTYPE_p_unsigned_char bs2, long n1, long n2, int ht, long ncodes, LongVector idx, SWIGTYPE_p_int dis) { return swigfaissJNI.match_hamming_thres(SWIGTYPE_p_unsigned_char.getCPtr(bs1), SWIGTYPE_p_unsigned_char.getCPtr(bs2), n1, n2, ht, ncodes, SWIGTYPE_p_long_long.getCPtr(idx.data()), idx, SWIGTYPE_p_int.getCPtr(dis)); } public static void crosshamming_count_thres(SWIGTYPE_p_unsigned_char dbs, long n, int ht, long ncodes, SWIGTYPE_p_unsigned_long nptr) { swigfaissJNI.crosshamming_count_thres(SWIGTYPE_p_unsigned_char.getCPtr(dbs), n, ht, ncodes, SWIGTYPE_p_unsigned_long.getCPtr(nptr)); } public static int get_num_gpus() { return swigfaissJNI.get_num_gpus(); } public static String get_compile_options() { return swigfaissJNI.get_compile_options(); } public static double getmillisecs() { return swigfaissJNI.getmillisecs(); } public static long get_mem_usage_kb() { return swigfaissJNI.get_mem_usage_kb(); } public static long get_cycles() { return swigfaissJNI.get_cycles(); } public static void fvec_madd(long n, SWIGTYPE_p_float a, float bf, SWIGTYPE_p_float b, SWIGTYPE_p_float c) { swigfaissJNI.fvec_madd(n, SWIGTYPE_p_float.getCPtr(a), bf, SWIGTYPE_p_float.getCPtr(b), SWIGTYPE_p_float.getCPtr(c)); } public static int fvec_madd_and_argmin(long n, SWIGTYPE_p_float a, float bf, SWIGTYPE_p_float b, SWIGTYPE_p_float c) { return swigfaissJNI.fvec_madd_and_argmin(n, SWIGTYPE_p_float.getCPtr(a), bf, SWIGTYPE_p_float.getCPtr(b), SWIGTYPE_p_float.getCPtr(c)); } public static void reflection(SWIGTYPE_p_float u, SWIGTYPE_p_float x, long n, long d, long nu) { swigfaissJNI.reflection(SWIGTYPE_p_float.getCPtr(u), SWIGTYPE_p_float.getCPtr(x), n, d, nu); } public static void matrix_qr(int m, int n, SWIGTYPE_p_float a) { swigfaissJNI.matrix_qr(m, n, SWIGTYPE_p_float.getCPtr(a)); } public static void ranklist_handle_ties(int k, LongVector idx, SWIGTYPE_p_float dis) { swigfaissJNI.ranklist_handle_ties(k, SWIGTYPE_p_long_long.getCPtr(idx.data()), idx, SWIGTYPE_p_float.getCPtr(dis)); } public static long ranklist_intersection_size(long k1, LongVector v1, long k2, LongVector v2) { return swigfaissJNI.ranklist_intersection_size(k1, SWIGTYPE_p_long_long.getCPtr(v1.data()), v1, k2, SWIGTYPE_p_long_long.getCPtr(v2.data()), v2); } public static long merge_result_table_with(long n, long k, LongVector I0, SWIGTYPE_p_float D0, LongVector I1, SWIGTYPE_p_float D1, boolean keep_min, long translation) { return swigfaissJNI.merge_result_table_with__SWIG_0(n, k, SWIGTYPE_p_long_long.getCPtr(I0.data()), I0, SWIGTYPE_p_float.getCPtr(D0), SWIGTYPE_p_long_long.getCPtr(I1.data()), I1, SWIGTYPE_p_float.getCPtr(D1), keep_min, translation); } public static long merge_result_table_with(long n, long k, LongVector I0, SWIGTYPE_p_float D0, LongVector I1, SWIGTYPE_p_float D1, boolean keep_min) { return swigfaissJNI.merge_result_table_with__SWIG_1(n, k, SWIGTYPE_p_long_long.getCPtr(I0.data()), I0, SWIGTYPE_p_float.getCPtr(D0), SWIGTYPE_p_long_long.getCPtr(I1.data()), I1, SWIGTYPE_p_float.getCPtr(D1), keep_min); } public static long merge_result_table_with(long n, long k, LongVector I0, SWIGTYPE_p_float D0, LongVector I1, SWIGTYPE_p_float D1) { return swigfaissJNI.merge_result_table_with__SWIG_2(n, k, SWIGTYPE_p_long_long.getCPtr(I0.data()), I0, SWIGTYPE_p_float.getCPtr(D0), SWIGTYPE_p_long_long.getCPtr(I1.data()), I1, SWIGTYPE_p_float.getCPtr(D1)); } public static double imbalance_factor(int n, int k, LongVector assign) { return swigfaissJNI.imbalance_factor__SWIG_0(n, k, SWIGTYPE_p_long_long.getCPtr(assign.data()), assign); } public static double imbalance_factor(int k, SWIGTYPE_p_int hist) { return swigfaissJNI.imbalance_factor__SWIG_1(k, SWIGTYPE_p_int.getCPtr(hist)); } public static void fvec_argsort(long n, SWIGTYPE_p_float vals, SWIGTYPE_p_unsigned_long perm) { swigfaissJNI.fvec_argsort(n, SWIGTYPE_p_float.getCPtr(vals), SWIGTYPE_p_unsigned_long.getCPtr(perm)); } public static void fvec_argsort_parallel(long n, SWIGTYPE_p_float vals, SWIGTYPE_p_unsigned_long perm) { swigfaissJNI.fvec_argsort_parallel(n, SWIGTYPE_p_float.getCPtr(vals), SWIGTYPE_p_unsigned_long.getCPtr(perm)); } public static int ivec_hist(long n, SWIGTYPE_p_int v, int vmax, SWIGTYPE_p_int hist) { return swigfaissJNI.ivec_hist(n, SWIGTYPE_p_int.getCPtr(v), vmax, SWIGTYPE_p_int.getCPtr(hist)); } public static void bincode_hist(long n, long nbits, SWIGTYPE_p_unsigned_char codes, SWIGTYPE_p_int hist) { swigfaissJNI.bincode_hist(n, nbits, SWIGTYPE_p_unsigned_char.getCPtr(codes), SWIGTYPE_p_int.getCPtr(hist)); } public static long ivec_checksum(long n, SWIGTYPE_p_int a) { return swigfaissJNI.ivec_checksum(n, SWIGTYPE_p_int.getCPtr(a)); } public static SWIGTYPE_p_float fvecs_maybe_subsample(long d, SWIGTYPE_p_unsigned_long n, long nmax, SWIGTYPE_p_float x, boolean verbose, long seed) { long cPtr = swigfaissJNI.fvecs_maybe_subsample__SWIG_0(d, SWIGTYPE_p_unsigned_long.getCPtr(n), nmax, SWIGTYPE_p_float.getCPtr(x), verbose, seed); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } public static SWIGTYPE_p_float fvecs_maybe_subsample(long d, SWIGTYPE_p_unsigned_long n, long nmax, SWIGTYPE_p_float x, boolean verbose) { long cPtr = swigfaissJNI.fvecs_maybe_subsample__SWIG_1(d, SWIGTYPE_p_unsigned_long.getCPtr(n), nmax, SWIGTYPE_p_float.getCPtr(x), verbose); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } public static SWIGTYPE_p_float fvecs_maybe_subsample(long d, SWIGTYPE_p_unsigned_long n, long nmax, SWIGTYPE_p_float x) { long cPtr = swigfaissJNI.fvecs_maybe_subsample__SWIG_2(d, SWIGTYPE_p_unsigned_long.getCPtr(n), nmax, SWIGTYPE_p_float.getCPtr(x)); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } public static void binary_to_real(long d, SWIGTYPE_p_unsigned_char x_in, SWIGTYPE_p_float x_out) { swigfaissJNI.binary_to_real(d, SWIGTYPE_p_unsigned_char.getCPtr(x_in), SWIGTYPE_p_float.getCPtr(x_out)); } public static void real_to_binary(long d, SWIGTYPE_p_float x_in, SWIGTYPE_p_unsigned_char x_out) { swigfaissJNI.real_to_binary(d, SWIGTYPE_p_float.getCPtr(x_in), SWIGTYPE_p_unsigned_char.getCPtr(x_out)); } public static long hash_bytes(SWIGTYPE_p_unsigned_char bytes, long n) { return swigfaissJNI.hash_bytes(SWIGTYPE_p_unsigned_char.getCPtr(bytes), n); } public static boolean check_openmp() { return swigfaissJNI.check_openmp(); } public static float kmeans_clustering(long d, long n, long k, SWIGTYPE_p_float x, SWIGTYPE_p_float centroids) { return swigfaissJNI.kmeans_clustering(d, n, k, SWIGTYPE_p_float.getCPtr(x), SWIGTYPE_p_float.getCPtr(centroids)); } public static void setIndexPQ_stats(IndexPQStats value) { swigfaissJNI.indexPQ_stats_set(IndexPQStats.getCPtr(value), value); } public static IndexPQStats getIndexPQ_stats() { long cPtr = swigfaissJNI.indexPQ_stats_get(); return (cPtr == 0) ? null : new IndexPQStats(cPtr, false); } public static void setIndexIVF_stats(IndexIVFStats value) { swigfaissJNI.indexIVF_stats_set(IndexIVFStats.getCPtr(value), value); } public static IndexIVFStats getIndexIVF_stats() { long cPtr = swigfaissJNI.indexIVF_stats_get(); return (cPtr == 0) ? null : new IndexIVFStats(cPtr, false); } public static short[] getHamdis_tab_ham_bytes() { return swigfaissJNI.hamdis_tab_ham_bytes_get(); } public static int generalized_hamming_64(long a) { return swigfaissJNI.generalized_hamming_64(a); } public static void generalized_hammings_knn_hc(SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_int_int64_t_t_t ha, SWIGTYPE_p_unsigned_char a, SWIGTYPE_p_unsigned_char b, long nb, long code_size, int ordered) { swigfaissJNI.generalized_hammings_knn_hc__SWIG_0(SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_int_int64_t_t_t.getCPtr(ha), SWIGTYPE_p_unsigned_char.getCPtr(a), SWIGTYPE_p_unsigned_char.getCPtr(b), nb, code_size, ordered); } public static void generalized_hammings_knn_hc(SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_int_int64_t_t_t ha, SWIGTYPE_p_unsigned_char a, SWIGTYPE_p_unsigned_char b, long nb, long code_size) { swigfaissJNI.generalized_hammings_knn_hc__SWIG_1(SWIGTYPE_p_faiss__HeapArrayT_faiss__CMaxT_int_int64_t_t_t.getCPtr(ha), SWIGTYPE_p_unsigned_char.getCPtr(a), SWIGTYPE_p_unsigned_char.getCPtr(b), nb, code_size); } public static void check_compatible_for_merge(Index index1, Index index2) { swigfaissJNI.check_compatible_for_merge(Index.getCPtr(index1), index1, Index.getCPtr(index2), index2); } public static IndexIVF extract_index_ivf(Index index) { long cPtr = swigfaissJNI.extract_index_ivf__SWIG_0(Index.getCPtr(index), index); return (cPtr == 0) ? null : new IndexIVF(cPtr, false); } public static IndexIVF try_extract_index_ivf(Index index) { long cPtr = swigfaissJNI.try_extract_index_ivf__SWIG_0(Index.getCPtr(index), index); return (cPtr == 0) ? null : new IndexIVF(cPtr, false); } public static void merge_into(Index index0, Index index1, boolean shift_ids) { swigfaissJNI.merge_into(Index.getCPtr(index0), index0, Index.getCPtr(index1), index1, shift_ids); } public static void search_centroid(Index index, SWIGTYPE_p_float x, int n, LongVector centroid_ids) { swigfaissJNI.search_centroid(Index.getCPtr(index), index, SWIGTYPE_p_float.getCPtr(x), n, SWIGTYPE_p_long_long.getCPtr(centroid_ids.data()), centroid_ids); } public static void search_and_return_centroids(Index index, long n, SWIGTYPE_p_float xin, int k, SWIGTYPE_p_float distances, LongVector labels, LongVector query_centroid_ids, LongVector result_centroid_ids) { swigfaissJNI.search_and_return_centroids(Index.getCPtr(index), index, n, SWIGTYPE_p_float.getCPtr(xin), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, SWIGTYPE_p_long_long.getCPtr(query_centroid_ids.data()), query_centroid_ids, SWIGTYPE_p_long_long.getCPtr(result_centroid_ids.data()), result_centroid_ids); } public static ArrayInvertedLists get_invlist_range(Index index, int i0, int i1) { long cPtr = swigfaissJNI.get_invlist_range(Index.getCPtr(index), index, i0, i1); return (cPtr == 0) ? null : new ArrayInvertedLists(cPtr, false); } public static void set_invlist_range(Index index, int i0, int i1, ArrayInvertedLists src) { swigfaissJNI.set_invlist_range(Index.getCPtr(index), index, i0, i1, ArrayInvertedLists.getCPtr(src), src); } public static void search_with_parameters(Index index, long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels, IVFSearchParameters params, SWIGTYPE_p_unsigned_long nb_dis, SWIGTYPE_p_double ms_per_stage) { swigfaissJNI.search_with_parameters__SWIG_0(Index.getCPtr(index), index, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, IVFSearchParameters.getCPtr(params), params, SWIGTYPE_p_unsigned_long.getCPtr(nb_dis), SWIGTYPE_p_double.getCPtr(ms_per_stage)); } public static void search_with_parameters(Index index, long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels, IVFSearchParameters params, SWIGTYPE_p_unsigned_long nb_dis) { swigfaissJNI.search_with_parameters__SWIG_1(Index.getCPtr(index), index, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, IVFSearchParameters.getCPtr(params), params, SWIGTYPE_p_unsigned_long.getCPtr(nb_dis)); } public static void search_with_parameters(Index index, long n, SWIGTYPE_p_float x, long k, SWIGTYPE_p_float distances, LongVector labels, IVFSearchParameters params) { swigfaissJNI.search_with_parameters__SWIG_2(Index.getCPtr(index), index, n, SWIGTYPE_p_float.getCPtr(x), k, SWIGTYPE_p_float.getCPtr(distances), SWIGTYPE_p_long_long.getCPtr(labels.data()), labels, IVFSearchParameters.getCPtr(params), params); } public static void range_search_with_parameters(Index index, long n, SWIGTYPE_p_float x, float radius, RangeSearchResult result, IVFSearchParameters params, SWIGTYPE_p_unsigned_long nb_dis, SWIGTYPE_p_double ms_per_stage) { swigfaissJNI.range_search_with_parameters__SWIG_0(Index.getCPtr(index), index, n, SWIGTYPE_p_float.getCPtr(x), radius, RangeSearchResult.getCPtr(result), result, IVFSearchParameters.getCPtr(params), params, SWIGTYPE_p_unsigned_long.getCPtr(nb_dis), SWIGTYPE_p_double.getCPtr(ms_per_stage)); } public static void range_search_with_parameters(Index index, long n, SWIGTYPE_p_float x, float radius, RangeSearchResult result, IVFSearchParameters params, SWIGTYPE_p_unsigned_long nb_dis) { swigfaissJNI.range_search_with_parameters__SWIG_1(Index.getCPtr(index), index, n, SWIGTYPE_p_float.getCPtr(x), radius, RangeSearchResult.getCPtr(result), result, IVFSearchParameters.getCPtr(params), params, SWIGTYPE_p_unsigned_long.getCPtr(nb_dis)); } public static void range_search_with_parameters(Index index, long n, SWIGTYPE_p_float x, float radius, RangeSearchResult result, IVFSearchParameters params) { swigfaissJNI.range_search_with_parameters__SWIG_2(Index.getCPtr(index), index, n, SWIGTYPE_p_float.getCPtr(x), radius, RangeSearchResult.getCPtr(result), result, IVFSearchParameters.getCPtr(params), params); } public static void setHnsw_stats(HNSWStats value) { swigfaissJNI.hnsw_stats_set(HNSWStats.getCPtr(value), value); } public static HNSWStats getHnsw_stats() { long cPtr = swigfaissJNI.hnsw_stats_get(); return (cPtr == 0) ? null : new HNSWStats(cPtr, false); } public static void setPrecomputed_table_max_bytes(long value) { swigfaissJNI.precomputed_table_max_bytes_set(value); } public static long getPrecomputed_table_max_bytes() { return swigfaissJNI.precomputed_table_max_bytes_get(); } public static void initialize_IVFPQ_precomputed_table(SWIGTYPE_p_int use_precomputed_table, Index quantizer, ProductQuantizer pq, SWIGTYPE_p_AlignedTableT_float_32_t precomputed_table, boolean verbose) { swigfaissJNI.initialize_IVFPQ_precomputed_table(SWIGTYPE_p_int.getCPtr(use_precomputed_table), Index.getCPtr(quantizer), quantizer, ProductQuantizer.getCPtr(pq), pq, SWIGTYPE_p_AlignedTableT_float_32_t.getCPtr(precomputed_table), verbose); } public static void setIndexIVFPQ_stats(IndexIVFPQStats value) { swigfaissJNI.indexIVFPQ_stats_set(IndexIVFPQStats.getCPtr(value), value); } public static IndexIVFPQStats getIndexIVFPQ_stats() { long cPtr = swigfaissJNI.indexIVFPQ_stats_get(); return (cPtr == 0) ? null : new IndexIVFPQStats(cPtr, false); } public static Index downcast_index(Index index) { long cPtr = swigfaissJNI.downcast_index(Index.getCPtr(index), index); return (cPtr == 0) ? null : new Index(cPtr, false); } public static VectorTransform downcast_VectorTransform(VectorTransform vt) { long cPtr = swigfaissJNI.downcast_VectorTransform(VectorTransform.getCPtr(vt), vt); return (cPtr == 0) ? null : new VectorTransform(cPtr, false); } public static IndexBinary downcast_IndexBinary(IndexBinary index) { long cPtr = swigfaissJNI.downcast_IndexBinary(IndexBinary.getCPtr(index), index); return (cPtr == 0) ? null : new IndexBinary(cPtr, false); } public static Index upcast_IndexShards(IndexShards index) { long cPtr = swigfaissJNI.upcast_IndexShards(IndexShards.getCPtr(index), index); return (cPtr == 0) ? null : new Index(cPtr, false); } public static void write_index(Index idx, String fname) { swigfaissJNI.write_index__SWIG_0(Index.getCPtr(idx), idx, fname); } public static void write_index(Index idx, SWIGTYPE_p_FILE f) { swigfaissJNI.write_index__SWIG_1(Index.getCPtr(idx), idx, SWIGTYPE_p_FILE.getCPtr(f)); } public static void write_index(Index idx, SWIGTYPE_p_faiss__IOWriter writer) { swigfaissJNI.write_index__SWIG_2(Index.getCPtr(idx), idx, SWIGTYPE_p_faiss__IOWriter.getCPtr(writer)); } public static void write_index_binary(IndexBinary idx, String fname) { swigfaissJNI.write_index_binary__SWIG_0(IndexBinary.getCPtr(idx), idx, fname); } public static void write_index_binary(IndexBinary idx, SWIGTYPE_p_FILE f) { swigfaissJNI.write_index_binary__SWIG_1(IndexBinary.getCPtr(idx), idx, SWIGTYPE_p_FILE.getCPtr(f)); } public static void write_index_binary(IndexBinary idx, SWIGTYPE_p_faiss__IOWriter writer) { swigfaissJNI.write_index_binary__SWIG_2(IndexBinary.getCPtr(idx), idx, SWIGTYPE_p_faiss__IOWriter.getCPtr(writer)); } public static int getIO_FLAG_READ_ONLY() { return swigfaissJNI.IO_FLAG_READ_ONLY_get(); } public static int getIO_FLAG_ONDISK_SAME_DIR() { return swigfaissJNI.IO_FLAG_ONDISK_SAME_DIR_get(); } public static int getIO_FLAG_SKIP_IVF_DATA() { return swigfaissJNI.IO_FLAG_SKIP_IVF_DATA_get(); } public static int getIO_FLAG_MMAP() { return swigfaissJNI.IO_FLAG_MMAP_get(); } public static Index read_index(String fname, int io_flags) { long cPtr = swigfaissJNI.read_index__SWIG_0(fname, io_flags); return (cPtr == 0) ? null : new Index(cPtr, true); } public static Index read_index(String fname) { long cPtr = swigfaissJNI.read_index__SWIG_1(fname); return (cPtr == 0) ? null : new Index(cPtr, true); } public static Index read_index(SWIGTYPE_p_FILE f, int io_flags) { long cPtr = swigfaissJNI.read_index__SWIG_2(SWIGTYPE_p_FILE.getCPtr(f), io_flags); return (cPtr == 0) ? null : new Index(cPtr, true); } public static Index read_index(SWIGTYPE_p_FILE f) { long cPtr = swigfaissJNI.read_index__SWIG_3(SWIGTYPE_p_FILE.getCPtr(f)); return (cPtr == 0) ? null : new Index(cPtr, true); } public static Index read_index(SWIGTYPE_p_faiss__IOReader reader, int io_flags) { long cPtr = swigfaissJNI.read_index__SWIG_4(SWIGTYPE_p_faiss__IOReader.getCPtr(reader), io_flags); return (cPtr == 0) ? null : new Index(cPtr, true); } public static Index read_index(SWIGTYPE_p_faiss__IOReader reader) { long cPtr = swigfaissJNI.read_index__SWIG_5(SWIGTYPE_p_faiss__IOReader.getCPtr(reader)); return (cPtr == 0) ? null : new Index(cPtr, true); } public static IndexBinary read_index_binary(String fname, int io_flags) { long cPtr = swigfaissJNI.read_index_binary__SWIG_0(fname, io_flags); return (cPtr == 0) ? null : new IndexBinary(cPtr, true); } public static IndexBinary read_index_binary(String fname) { long cPtr = swigfaissJNI.read_index_binary__SWIG_1(fname); return (cPtr == 0) ? null : new IndexBinary(cPtr, true); } public static IndexBinary read_index_binary(SWIGTYPE_p_FILE f, int io_flags) { long cPtr = swigfaissJNI.read_index_binary__SWIG_2(SWIGTYPE_p_FILE.getCPtr(f), io_flags); return (cPtr == 0) ? null : new IndexBinary(cPtr, true); } public static IndexBinary read_index_binary(SWIGTYPE_p_FILE f) { long cPtr = swigfaissJNI.read_index_binary__SWIG_3(SWIGTYPE_p_FILE.getCPtr(f)); return (cPtr == 0) ? null : new IndexBinary(cPtr, true); } public static IndexBinary read_index_binary(SWIGTYPE_p_faiss__IOReader reader, int io_flags) { long cPtr = swigfaissJNI.read_index_binary__SWIG_4(SWIGTYPE_p_faiss__IOReader.getCPtr(reader), io_flags); return (cPtr == 0) ? null : new IndexBinary(cPtr, true); } public static IndexBinary read_index_binary(SWIGTYPE_p_faiss__IOReader reader) { long cPtr = swigfaissJNI.read_index_binary__SWIG_5(SWIGTYPE_p_faiss__IOReader.getCPtr(reader)); return (cPtr == 0) ? null : new IndexBinary(cPtr, true); } public static void write_VectorTransform(VectorTransform vt, String fname) { swigfaissJNI.write_VectorTransform(VectorTransform.getCPtr(vt), vt, fname); } public static VectorTransform read_VectorTransform(String fname) { long cPtr = swigfaissJNI.read_VectorTransform(fname); return (cPtr == 0) ? null : new VectorTransform(cPtr, true); } public static ProductQuantizer read_ProductQuantizer(String fname) { long cPtr = swigfaissJNI.read_ProductQuantizer__SWIG_0(fname); return (cPtr == 0) ? null : new ProductQuantizer(cPtr, true); } public static ProductQuantizer read_ProductQuantizer(SWIGTYPE_p_faiss__IOReader reader) { long cPtr = swigfaissJNI.read_ProductQuantizer__SWIG_1(SWIGTYPE_p_faiss__IOReader.getCPtr(reader)); return (cPtr == 0) ? null : new ProductQuantizer(cPtr, true); } public static void write_ProductQuantizer(ProductQuantizer pq, String fname) { swigfaissJNI.write_ProductQuantizer__SWIG_0(ProductQuantizer.getCPtr(pq), pq, fname); } public static void write_ProductQuantizer(ProductQuantizer pq, SWIGTYPE_p_faiss__IOWriter f) { swigfaissJNI.write_ProductQuantizer__SWIG_1(ProductQuantizer.getCPtr(pq), pq, SWIGTYPE_p_faiss__IOWriter.getCPtr(f)); } public static void write_InvertedLists(InvertedLists ils, SWIGTYPE_p_faiss__IOWriter f) { swigfaissJNI.write_InvertedLists(InvertedLists.getCPtr(ils), ils, SWIGTYPE_p_faiss__IOWriter.getCPtr(f)); } public static InvertedLists read_InvertedLists(SWIGTYPE_p_faiss__IOReader reader, int io_flags) { long cPtr = swigfaissJNI.read_InvertedLists__SWIG_0(SWIGTYPE_p_faiss__IOReader.getCPtr(reader), io_flags); return (cPtr == 0) ? null : new InvertedLists(cPtr, false); } public static InvertedLists read_InvertedLists(SWIGTYPE_p_faiss__IOReader reader) { long cPtr = swigfaissJNI.read_InvertedLists__SWIG_1(SWIGTYPE_p_faiss__IOReader.getCPtr(reader)); return (cPtr == 0) ? null : new InvertedLists(cPtr, false); } public static Index index_factory(int d, String description, MetricType metric) { long cPtr = swigfaissJNI.index_factory__SWIG_0(d, description, metric.swigValue()); return (cPtr == 0) ? null : new Index(cPtr, true); } public static Index index_factory(int d, String description) { long cPtr = swigfaissJNI.index_factory__SWIG_1(d, description); return (cPtr == 0) ? null : new Index(cPtr, true); } public static void setIndex_factory_verbose(int value) { swigfaissJNI.index_factory_verbose_set(value); } public static int getIndex_factory_verbose() { return swigfaissJNI.index_factory_verbose_get(); } public static IndexBinary index_binary_factory(int d, String description) { long cPtr = swigfaissJNI.index_binary_factory(d, description); return (cPtr == 0) ? null : new IndexBinary(cPtr, true); } public static void simd_histogram_8(SWIGTYPE_p_uint16_t data, int n, SWIGTYPE_p_uint16_t min, int shift, SWIGTYPE_p_int hist) { swigfaissJNI.simd_histogram_8(SWIGTYPE_p_uint16_t.getCPtr(data), n, SWIGTYPE_p_uint16_t.getCPtr(min), shift, SWIGTYPE_p_int.getCPtr(hist)); } public static void simd_histogram_16(SWIGTYPE_p_uint16_t data, int n, SWIGTYPE_p_uint16_t min, int shift, SWIGTYPE_p_int hist) { swigfaissJNI.simd_histogram_16(SWIGTYPE_p_uint16_t.getCPtr(data), n, SWIGTYPE_p_uint16_t.getCPtr(min), shift, SWIGTYPE_p_int.getCPtr(hist)); } public static void setPartition_stats(PartitionStats value) { swigfaissJNI.partition_stats_set(PartitionStats.getCPtr(value), value); } public static PartitionStats getPartition_stats() { long cPtr = swigfaissJNI.partition_stats_get(); return (cPtr == 0) ? null : new PartitionStats(cPtr, false); } public static float CMin_float_partition_fuzzy(SWIGTYPE_p_float vals, LongVector ids, long n, long q_min, long q_max, SWIGTYPE_p_unsigned_long q_out) { return swigfaissJNI.CMin_float_partition_fuzzy(SWIGTYPE_p_float.getCPtr(vals), SWIGTYPE_p_long_long.getCPtr(ids.data()), ids, n, q_min, q_max, SWIGTYPE_p_unsigned_long.getCPtr(q_out)); } public static float CMax_float_partition_fuzzy(SWIGTYPE_p_float vals, LongVector ids, long n, long q_min, long q_max, SWIGTYPE_p_unsigned_long q_out) { return swigfaissJNI.CMax_float_partition_fuzzy(SWIGTYPE_p_float.getCPtr(vals), SWIGTYPE_p_long_long.getCPtr(ids.data()), ids, n, q_min, q_max, SWIGTYPE_p_unsigned_long.getCPtr(q_out)); } public static SWIGTYPE_p_uint16_t CMax_uint16_partition_fuzzy(SWIGTYPE_p_uint16_t vals, LongVector ids, long n, long q_min, long q_max, SWIGTYPE_p_unsigned_long q_out) { return new SWIGTYPE_p_uint16_t(swigfaissJNI.CMax_uint16_partition_fuzzy__SWIG_0(SWIGTYPE_p_uint16_t.getCPtr(vals), SWIGTYPE_p_long_long.getCPtr(ids.data()), ids, n, q_min, q_max, SWIGTYPE_p_unsigned_long.getCPtr(q_out)), true); } public static SWIGTYPE_p_uint16_t CMin_uint16_partition_fuzzy(SWIGTYPE_p_uint16_t vals, LongVector ids, long n, long q_min, long q_max, SWIGTYPE_p_unsigned_long q_out) { return new SWIGTYPE_p_uint16_t(swigfaissJNI.CMin_uint16_partition_fuzzy__SWIG_0(SWIGTYPE_p_uint16_t.getCPtr(vals), SWIGTYPE_p_long_long.getCPtr(ids.data()), ids, n, q_min, q_max, SWIGTYPE_p_unsigned_long.getCPtr(q_out)), true); } public static SWIGTYPE_p_uint16_t CMax_uint16_partition_fuzzy(SWIGTYPE_p_uint16_t vals, SWIGTYPE_p_int ids, long n, long q_min, long q_max, SWIGTYPE_p_unsigned_long q_out) { return new SWIGTYPE_p_uint16_t(swigfaissJNI.CMax_uint16_partition_fuzzy__SWIG_1(SWIGTYPE_p_uint16_t.getCPtr(vals), SWIGTYPE_p_int.getCPtr(ids), n, q_min, q_max, SWIGTYPE_p_unsigned_long.getCPtr(q_out)), true); } public static SWIGTYPE_p_uint16_t CMin_uint16_partition_fuzzy(SWIGTYPE_p_uint16_t vals, SWIGTYPE_p_int ids, long n, long q_min, long q_max, SWIGTYPE_p_unsigned_long q_out) { return new SWIGTYPE_p_uint16_t(swigfaissJNI.CMin_uint16_partition_fuzzy__SWIG_1(SWIGTYPE_p_uint16_t.getCPtr(vals), SWIGTYPE_p_int.getCPtr(ids), n, q_min, q_max, SWIGTYPE_p_unsigned_long.getCPtr(q_out)), true); } public static void omp_set_num_threads(int num_threads) { swigfaissJNI.omp_set_num_threads(num_threads); } public static int omp_get_max_threads() { return swigfaissJNI.omp_get_max_threads(); } public static SWIGTYPE_p_void memcpy(SWIGTYPE_p_void dest, SWIGTYPE_p_void src, long n) { long cPtr = swigfaissJNI.memcpy(SWIGTYPE_p_void.getCPtr(dest), SWIGTYPE_p_void.getCPtr(src), n); return (cPtr == 0) ? null : new SWIGTYPE_p_void(cPtr, false); } public static SWIGTYPE_p_float cast_integer_to_float_ptr(int x) { long cPtr = swigfaissJNI.cast_integer_to_float_ptr(x); return (cPtr == 0) ? null : new SWIGTYPE_p_float(cPtr, false); } public static SWIGTYPE_p_long cast_integer_to_long_ptr(int x) { long cPtr = swigfaissJNI.cast_integer_to_long_ptr(x); return (cPtr == 0) ? null : new SWIGTYPE_p_long(cPtr, false); } public static SWIGTYPE_p_int cast_integer_to_int_ptr(int x) { long cPtr = swigfaissJNI.cast_integer_to_int_ptr(x); return (cPtr == 0) ? null : new SWIGTYPE_p_int(cPtr, false); } public static void ignore_SIGTTIN() { swigfaissJNI.ignore_SIGTTIN(); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/swigfaissConstants.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; public interface swigfaissConstants { public final static int FAISS_VERSION_MAJOR = swigfaissJNI.FAISS_VERSION_MAJOR_get(); public final static int FAISS_VERSION_MINOR = swigfaissJNI.FAISS_VERSION_MINOR_get(); public final static int FAISS_VERSION_PATCH = swigfaissJNI.FAISS_VERSION_PATCH_get(); } ================================================ FILE: ann/src/main/java/com/twitter/ann/faiss/swig/swigfaissJNI.java ================================================ /* ---------------------------------------------------------------------------- * This file was automatically generated by SWIG (http://www.swig.org). * Version 4.0.2 * * Do not make changes to this file unless you know what you are doing--modify * the SWIG interface file instead. * ----------------------------------------------------------------------------- */ package com.twitter.ann.faiss; import com.twitter.ann.faiss.NativeUtils; public class swigfaissJNI { static { try { if (NativeUtils.getOperatingSystemType() == NativeUtils.OSType.MacOS) { NativeUtils.loadLibraryFromJar("/com/twitter/ann/faiss/swig/resources/swigfaiss.dylib"); } else { NativeUtils.loadLibraryFromJar("/com/twitter/ann/faiss/swig/resources/libstdc++.so.6"); NativeUtils.loadLibraryFromJar("/com/twitter/ann/faiss/swig/resources/libgcc_s.so.1"); NativeUtils.loadLibraryFromJar("/com/twitter/ann/faiss/swig/resources/libgomp.so.1"); NativeUtils.loadLibraryFromJar("/com/twitter/ann/faiss/swig/resources/libquadmath.so.0"); NativeUtils.loadLibraryFromJar("/com/twitter/ann/faiss/swig/resources/libgfortran.so.5"); NativeUtils.loadLibraryFromJar("/com/twitter/ann/faiss/swig/resources/swigfaiss.so"); } } catch (Exception e) { System.err.println("Native code library failed to load. \n" + e); System.exit(1); } } public final static native long new_intArray(int jarg1); public final static native void delete_intArray(long jarg1); public final static native int intArray_getitem(long jarg1, intArray jarg1_, int jarg2); public final static native void intArray_setitem(long jarg1, intArray jarg1_, int jarg2, int jarg3); public final static native long intArray_cast(long jarg1, intArray jarg1_); public final static native long intArray_frompointer(long jarg1); public final static native long new_floatArray(int jarg1); public final static native void delete_floatArray(long jarg1); public final static native float floatArray_getitem(long jarg1, floatArray jarg1_, int jarg2); public final static native void floatArray_setitem(long jarg1, floatArray jarg1_, int jarg2, float jarg3); public final static native long floatArray_cast(long jarg1, floatArray jarg1_); public final static native long floatArray_frompointer(long jarg1); public final static native long new_longArray(int jarg1); public final static native void delete_longArray(long jarg1); public final static native long longArray_getitem(long jarg1, longArray jarg1_, int jarg2); public final static native void longArray_setitem(long jarg1, longArray jarg1_, int jarg2, long jarg3); public final static native long longArray_cast(long jarg1, longArray jarg1_); public final static native long longArray_frompointer(long jarg1); public final static native long new_doubleArray(int jarg1); public final static native void delete_doubleArray(long jarg1); public final static native double doubleArray_getitem(long jarg1, doubleArray jarg1_, int jarg2); public final static native void doubleArray_setitem(long jarg1, doubleArray jarg1_, int jarg2, double jarg3); public final static native long doubleArray_cast(long jarg1, doubleArray jarg1_); public final static native long doubleArray_frompointer(long jarg1); public final static native long new_FloatVector(); public final static native void FloatVector_push_back(long jarg1, FloatVector jarg1_, float jarg2); public final static native void FloatVector_clear(long jarg1, FloatVector jarg1_); public final static native long FloatVector_data(long jarg1, FloatVector jarg1_); public final static native long FloatVector_size(long jarg1, FloatVector jarg1_); public final static native float FloatVector_at(long jarg1, FloatVector jarg1_, long jarg2); public final static native void FloatVector_resize(long jarg1, FloatVector jarg1_, long jarg2); public final static native void FloatVector_reserve(long jarg1, FloatVector jarg1_, long jarg2); public final static native void FloatVector_swap(long jarg1, FloatVector jarg1_, long jarg2, FloatVector jarg2_); public final static native void delete_FloatVector(long jarg1); public final static native long new_DoubleVector(); public final static native void DoubleVector_push_back(long jarg1, DoubleVector jarg1_, double jarg2); public final static native void DoubleVector_clear(long jarg1, DoubleVector jarg1_); public final static native long DoubleVector_data(long jarg1, DoubleVector jarg1_); public final static native long DoubleVector_size(long jarg1, DoubleVector jarg1_); public final static native double DoubleVector_at(long jarg1, DoubleVector jarg1_, long jarg2); public final static native void DoubleVector_resize(long jarg1, DoubleVector jarg1_, long jarg2); public final static native void DoubleVector_reserve(long jarg1, DoubleVector jarg1_, long jarg2); public final static native void DoubleVector_swap(long jarg1, DoubleVector jarg1_, long jarg2, DoubleVector jarg2_); public final static native void delete_DoubleVector(long jarg1); public final static native long new_ByteVector(); public final static native void ByteVector_push_back(long jarg1, ByteVector jarg1_, short jarg2); public final static native void ByteVector_clear(long jarg1, ByteVector jarg1_); public final static native long ByteVector_data(long jarg1, ByteVector jarg1_); public final static native long ByteVector_size(long jarg1, ByteVector jarg1_); public final static native short ByteVector_at(long jarg1, ByteVector jarg1_, long jarg2); public final static native void ByteVector_resize(long jarg1, ByteVector jarg1_, long jarg2); public final static native void ByteVector_reserve(long jarg1, ByteVector jarg1_, long jarg2); public final static native void ByteVector_swap(long jarg1, ByteVector jarg1_, long jarg2, ByteVector jarg2_); public final static native void delete_ByteVector(long jarg1); public final static native long new_CharVector(); public final static native void CharVector_push_back(long jarg1, CharVector jarg1_, char jarg2); public final static native void CharVector_clear(long jarg1, CharVector jarg1_); public final static native String CharVector_data(long jarg1, CharVector jarg1_); public final static native long CharVector_size(long jarg1, CharVector jarg1_); public final static native char CharVector_at(long jarg1, CharVector jarg1_, long jarg2); public final static native void CharVector_resize(long jarg1, CharVector jarg1_, long jarg2); public final static native void CharVector_reserve(long jarg1, CharVector jarg1_, long jarg2); public final static native void CharVector_swap(long jarg1, CharVector jarg1_, long jarg2, CharVector jarg2_); public final static native void delete_CharVector(long jarg1); public final static native long new_Uint64Vector(); public final static native void Uint64Vector_push_back(long jarg1, Uint64Vector jarg1_, long jarg2); public final static native void Uint64Vector_clear(long jarg1, Uint64Vector jarg1_); public final static native long Uint64Vector_data(long jarg1, Uint64Vector jarg1_); public final static native long Uint64Vector_size(long jarg1, Uint64Vector jarg1_); public final static native long Uint64Vector_at(long jarg1, Uint64Vector jarg1_, long jarg2); public final static native void Uint64Vector_resize(long jarg1, Uint64Vector jarg1_, long jarg2); public final static native void Uint64Vector_reserve(long jarg1, Uint64Vector jarg1_, long jarg2); public final static native void Uint64Vector_swap(long jarg1, Uint64Vector jarg1_, long jarg2, Uint64Vector jarg2_); public final static native void delete_Uint64Vector(long jarg1); public final static native long new_LongVector(); public final static native void LongVector_push_back(long jarg1, LongVector jarg1_, long jarg2); public final static native void LongVector_clear(long jarg1, LongVector jarg1_); public final static native long LongVector_data(long jarg1, LongVector jarg1_); public final static native long LongVector_size(long jarg1, LongVector jarg1_); public final static native long LongVector_at(long jarg1, LongVector jarg1_, long jarg2); public final static native void LongVector_resize(long jarg1, LongVector jarg1_, long jarg2); public final static native void LongVector_reserve(long jarg1, LongVector jarg1_, long jarg2); public final static native void LongVector_swap(long jarg1, LongVector jarg1_, long jarg2, LongVector jarg2_); public final static native void delete_LongVector(long jarg1); public final static native long new_IntVector(); public final static native void IntVector_push_back(long jarg1, IntVector jarg1_, int jarg2); public final static native void IntVector_clear(long jarg1, IntVector jarg1_); public final static native long IntVector_data(long jarg1, IntVector jarg1_); public final static native long IntVector_size(long jarg1, IntVector jarg1_); public final static native int IntVector_at(long jarg1, IntVector jarg1_, long jarg2); public final static native void IntVector_resize(long jarg1, IntVector jarg1_, long jarg2); public final static native void IntVector_reserve(long jarg1, IntVector jarg1_, long jarg2); public final static native void IntVector_swap(long jarg1, IntVector jarg1_, long jarg2, IntVector jarg2_); public final static native void delete_IntVector(long jarg1); public final static native long new_VectorTransformVector(); public final static native void VectorTransformVector_push_back(long jarg1, VectorTransformVector jarg1_, long jarg2, VectorTransform jarg2_); public final static native void VectorTransformVector_clear(long jarg1, VectorTransformVector jarg1_); public final static native long VectorTransformVector_data(long jarg1, VectorTransformVector jarg1_); public final static native long VectorTransformVector_size(long jarg1, VectorTransformVector jarg1_); public final static native long VectorTransformVector_at(long jarg1, VectorTransformVector jarg1_, long jarg2); public final static native void VectorTransformVector_resize(long jarg1, VectorTransformVector jarg1_, long jarg2); public final static native void VectorTransformVector_reserve(long jarg1, VectorTransformVector jarg1_, long jarg2); public final static native void VectorTransformVector_swap(long jarg1, VectorTransformVector jarg1_, long jarg2, VectorTransformVector jarg2_); public final static native void delete_VectorTransformVector(long jarg1); public final static native long new_OperatingPointVector(); public final static native void OperatingPointVector_push_back(long jarg1, OperatingPointVector jarg1_, long jarg2, OperatingPoint jarg2_); public final static native void OperatingPointVector_clear(long jarg1, OperatingPointVector jarg1_); public final static native long OperatingPointVector_data(long jarg1, OperatingPointVector jarg1_); public final static native long OperatingPointVector_size(long jarg1, OperatingPointVector jarg1_); public final static native long OperatingPointVector_at(long jarg1, OperatingPointVector jarg1_, long jarg2); public final static native void OperatingPointVector_resize(long jarg1, OperatingPointVector jarg1_, long jarg2); public final static native void OperatingPointVector_reserve(long jarg1, OperatingPointVector jarg1_, long jarg2); public final static native void OperatingPointVector_swap(long jarg1, OperatingPointVector jarg1_, long jarg2, OperatingPointVector jarg2_); public final static native void delete_OperatingPointVector(long jarg1); public final static native long new_InvertedListsPtrVector(); public final static native void InvertedListsPtrVector_push_back(long jarg1, InvertedListsPtrVector jarg1_, long jarg2, InvertedLists jarg2_); public final static native void InvertedListsPtrVector_clear(long jarg1, InvertedListsPtrVector jarg1_); public final static native long InvertedListsPtrVector_data(long jarg1, InvertedListsPtrVector jarg1_); public final static native long InvertedListsPtrVector_size(long jarg1, InvertedListsPtrVector jarg1_); public final static native long InvertedListsPtrVector_at(long jarg1, InvertedListsPtrVector jarg1_, long jarg2); public final static native void InvertedListsPtrVector_resize(long jarg1, InvertedListsPtrVector jarg1_, long jarg2); public final static native void InvertedListsPtrVector_reserve(long jarg1, InvertedListsPtrVector jarg1_, long jarg2); public final static native void InvertedListsPtrVector_swap(long jarg1, InvertedListsPtrVector jarg1_, long jarg2, InvertedListsPtrVector jarg2_); public final static native void delete_InvertedListsPtrVector(long jarg1); public final static native long new_FloatVectorVector(); public final static native void FloatVectorVector_push_back(long jarg1, FloatVectorVector jarg1_, long jarg2, FloatVector jarg2_); public final static native void FloatVectorVector_clear(long jarg1, FloatVectorVector jarg1_); public final static native long FloatVectorVector_data(long jarg1, FloatVectorVector jarg1_); public final static native long FloatVectorVector_size(long jarg1, FloatVectorVector jarg1_); public final static native long FloatVectorVector_at(long jarg1, FloatVectorVector jarg1_, long jarg2); public final static native void FloatVectorVector_resize(long jarg1, FloatVectorVector jarg1_, long jarg2); public final static native void FloatVectorVector_reserve(long jarg1, FloatVectorVector jarg1_, long jarg2); public final static native void FloatVectorVector_swap(long jarg1, FloatVectorVector jarg1_, long jarg2, FloatVectorVector jarg2_); public final static native void delete_FloatVectorVector(long jarg1); public final static native long new_ByteVectorVector(); public final static native void ByteVectorVector_push_back(long jarg1, ByteVectorVector jarg1_, long jarg2, ByteVector jarg2_); public final static native void ByteVectorVector_clear(long jarg1, ByteVectorVector jarg1_); public final static native long ByteVectorVector_data(long jarg1, ByteVectorVector jarg1_); public final static native long ByteVectorVector_size(long jarg1, ByteVectorVector jarg1_); public final static native long ByteVectorVector_at(long jarg1, ByteVectorVector jarg1_, long jarg2); public final static native void ByteVectorVector_resize(long jarg1, ByteVectorVector jarg1_, long jarg2); public final static native void ByteVectorVector_reserve(long jarg1, ByteVectorVector jarg1_, long jarg2); public final static native void ByteVectorVector_swap(long jarg1, ByteVectorVector jarg1_, long jarg2, ByteVectorVector jarg2_); public final static native void delete_ByteVectorVector(long jarg1); public final static native long new_LongVectorVector(); public final static native void LongVectorVector_push_back(long jarg1, LongVectorVector jarg1_, long jarg2); public final static native void LongVectorVector_clear(long jarg1, LongVectorVector jarg1_); public final static native long LongVectorVector_data(long jarg1, LongVectorVector jarg1_); public final static native long LongVectorVector_size(long jarg1, LongVectorVector jarg1_); public final static native long LongVectorVector_at(long jarg1, LongVectorVector jarg1_, long jarg2); public final static native void LongVectorVector_resize(long jarg1, LongVectorVector jarg1_, long jarg2); public final static native void LongVectorVector_reserve(long jarg1, LongVectorVector jarg1_, long jarg2); public final static native void LongVectorVector_swap(long jarg1, LongVectorVector jarg1_, long jarg2, LongVectorVector jarg2_); public final static native void delete_LongVectorVector(long jarg1); public final static native void bitvec_print(long jarg1, long jarg2); public final static native void fvecs2bitvecs(long jarg1, long jarg2, long jarg3, long jarg4); public final static native void bitvecs2fvecs(long jarg1, long jarg2, long jarg3, long jarg4); public final static native void fvec2bitvec(long jarg1, long jarg2, long jarg3); public final static native void bitvec_shuffle(long jarg1, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6); public final static native void BitstringWriter_code_set(long jarg1, BitstringWriter jarg1_, long jarg2); public final static native long BitstringWriter_code_get(long jarg1, BitstringWriter jarg1_); public final static native void BitstringWriter_code_size_set(long jarg1, BitstringWriter jarg1_, long jarg2); public final static native long BitstringWriter_code_size_get(long jarg1, BitstringWriter jarg1_); public final static native void BitstringWriter_i_set(long jarg1, BitstringWriter jarg1_, long jarg2); public final static native long BitstringWriter_i_get(long jarg1, BitstringWriter jarg1_); public final static native long new_BitstringWriter(long jarg1, long jarg2); public final static native void BitstringWriter_write(long jarg1, BitstringWriter jarg1_, long jarg2, int jarg3); public final static native void delete_BitstringWriter(long jarg1); public final static native void BitstringReader_code_set(long jarg1, BitstringReader jarg1_, long jarg2); public final static native long BitstringReader_code_get(long jarg1, BitstringReader jarg1_); public final static native void BitstringReader_code_size_set(long jarg1, BitstringReader jarg1_, long jarg2); public final static native long BitstringReader_code_size_get(long jarg1, BitstringReader jarg1_); public final static native void BitstringReader_i_set(long jarg1, BitstringReader jarg1_, long jarg2); public final static native long BitstringReader_i_get(long jarg1, BitstringReader jarg1_); public final static native long new_BitstringReader(long jarg1, long jarg2); public final static native long BitstringReader_read(long jarg1, BitstringReader jarg1_, int jarg2); public final static native void delete_BitstringReader(long jarg1); public final static native void hamming_batch_size_set(long jarg1); public final static native long hamming_batch_size_get(); public final static native int popcount64(long jarg1); public final static native void hammings(long jarg1, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6); public final static native void hammings_knn_hc(long jarg1, long jarg2, long jarg3, long jarg4, long jarg5, int jarg6); public final static native void hammings_knn(long jarg1, long jarg2, long jarg3, long jarg4, long jarg5, int jarg6); public final static native void hammings_knn_mc(long jarg1, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, long jarg7, long jarg8, LongVector jarg8_); public final static native void hamming_range_search(long jarg1, long jarg2, long jarg3, long jarg4, int jarg5, long jarg6, long jarg7, RangeSearchResult jarg7_); public final static native void hamming_count_thres(long jarg1, long jarg2, long jarg3, long jarg4, int jarg5, long jarg6, long jarg7); public final static native long match_hamming_thres(long jarg1, long jarg2, long jarg3, long jarg4, int jarg5, long jarg6, long jarg7, LongVector jarg7_, long jarg8); public final static native void crosshamming_count_thres(long jarg1, long jarg2, int jarg3, long jarg4, long jarg5); public final static native int get_num_gpus(); public final static native int METRIC_INNER_PRODUCT_get(); public final static native int METRIC_L2_get(); public final static native int METRIC_Canberra_get(); public final static native String get_compile_options(); public final static native double getmillisecs(); public final static native long get_mem_usage_kb(); public final static native long get_cycles(); public final static native void fvec_madd(long jarg1, long jarg2, float jarg3, long jarg4, long jarg5); public final static native int fvec_madd_and_argmin(long jarg1, long jarg2, float jarg3, long jarg4, long jarg5); public final static native void reflection(long jarg1, long jarg2, long jarg3, long jarg4, long jarg5); public final static native void matrix_qr(int jarg1, int jarg2, long jarg3); public final static native void ranklist_handle_ties(int jarg1, long jarg2, LongVector jarg2_, long jarg3); public final static native long ranklist_intersection_size(long jarg1, long jarg2, LongVector jarg2_, long jarg3, long jarg4, LongVector jarg4_); public final static native long merge_result_table_with__SWIG_0(long jarg1, long jarg2, long jarg3, LongVector jarg3_, long jarg4, long jarg5, LongVector jarg5_, long jarg6, boolean jarg7, long jarg8); public final static native long merge_result_table_with__SWIG_1(long jarg1, long jarg2, long jarg3, LongVector jarg3_, long jarg4, long jarg5, LongVector jarg5_, long jarg6, boolean jarg7); public final static native long merge_result_table_with__SWIG_2(long jarg1, long jarg2, long jarg3, LongVector jarg3_, long jarg4, long jarg5, LongVector jarg5_, long jarg6); public final static native double imbalance_factor__SWIG_0(int jarg1, int jarg2, long jarg3, LongVector jarg3_); public final static native double imbalance_factor__SWIG_1(int jarg1, long jarg2); public final static native void fvec_argsort(long jarg1, long jarg2, long jarg3); public final static native void fvec_argsort_parallel(long jarg1, long jarg2, long jarg3); public final static native int ivec_hist(long jarg1, long jarg2, int jarg3, long jarg4); public final static native void bincode_hist(long jarg1, long jarg2, long jarg3, long jarg4); public final static native long ivec_checksum(long jarg1, long jarg2); public final static native long fvecs_maybe_subsample__SWIG_0(long jarg1, long jarg2, long jarg3, long jarg4, boolean jarg5, long jarg6); public final static native long fvecs_maybe_subsample__SWIG_1(long jarg1, long jarg2, long jarg3, long jarg4, boolean jarg5); public final static native long fvecs_maybe_subsample__SWIG_2(long jarg1, long jarg2, long jarg3, long jarg4); public final static native void binary_to_real(long jarg1, long jarg2, long jarg3); public final static native void real_to_binary(long jarg1, long jarg2, long jarg3); public final static native long hash_bytes(long jarg1, long jarg2); public final static native boolean check_openmp(); public final static native int FAISS_VERSION_MAJOR_get(); public final static native int FAISS_VERSION_MINOR_get(); public final static native int FAISS_VERSION_PATCH_get(); public final static native void Index_d_set(long jarg1, Index jarg1_, int jarg2); public final static native int Index_d_get(long jarg1, Index jarg1_); public final static native void Index_ntotal_set(long jarg1, Index jarg1_, long jarg2); public final static native long Index_ntotal_get(long jarg1, Index jarg1_); public final static native void Index_verbose_set(long jarg1, Index jarg1_, boolean jarg2); public final static native boolean Index_verbose_get(long jarg1, Index jarg1_); public final static native void Index_is_trained_set(long jarg1, Index jarg1_, boolean jarg2); public final static native boolean Index_is_trained_get(long jarg1, Index jarg1_); public final static native void Index_metric_type_set(long jarg1, Index jarg1_, int jarg2); public final static native int Index_metric_type_get(long jarg1, Index jarg1_); public final static native void Index_metric_arg_set(long jarg1, Index jarg1_, float jarg2); public final static native float Index_metric_arg_get(long jarg1, Index jarg1_); public final static native void delete_Index(long jarg1); public final static native void Index_train(long jarg1, Index jarg1_, long jarg2, long jarg3); public final static native void Index_add(long jarg1, Index jarg1_, long jarg2, long jarg3); public final static native void Index_add_with_ids(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_); public final static native void Index_search(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void Index_range_search(long jarg1, Index jarg1_, long jarg2, long jarg3, float jarg4, long jarg5, RangeSearchResult jarg5_); public final static native void Index_assign__SWIG_0(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5); public final static native void Index_assign__SWIG_1(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_); public final static native void Index_reset(long jarg1, Index jarg1_); public final static native long Index_remove_ids(long jarg1, Index jarg1_, long jarg2, IDSelector jarg2_); public final static native void Index_reconstruct(long jarg1, Index jarg1_, long jarg2, long jarg3); public final static native void Index_reconstruct_n(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4); public final static native void Index_search_and_reconstruct(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_, long jarg7); public final static native void Index_compute_residual(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4); public final static native void Index_compute_residual_n(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, LongVector jarg5_); public final static native long Index_get_distance_computer(long jarg1, Index jarg1_); public final static native long Index_sa_code_size(long jarg1, Index jarg1_); public final static native void Index_sa_encode(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4); public final static native void Index_sa_decode(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4); public final static native long Index_toIVF(long jarg1, Index jarg1_); public final static native void ClusteringParameters_niter_set(long jarg1, ClusteringParameters jarg1_, int jarg2); public final static native int ClusteringParameters_niter_get(long jarg1, ClusteringParameters jarg1_); public final static native void ClusteringParameters_nredo_set(long jarg1, ClusteringParameters jarg1_, int jarg2); public final static native int ClusteringParameters_nredo_get(long jarg1, ClusteringParameters jarg1_); public final static native void ClusteringParameters_verbose_set(long jarg1, ClusteringParameters jarg1_, boolean jarg2); public final static native boolean ClusteringParameters_verbose_get(long jarg1, ClusteringParameters jarg1_); public final static native void ClusteringParameters_spherical_set(long jarg1, ClusteringParameters jarg1_, boolean jarg2); public final static native boolean ClusteringParameters_spherical_get(long jarg1, ClusteringParameters jarg1_); public final static native void ClusteringParameters_int_centroids_set(long jarg1, ClusteringParameters jarg1_, boolean jarg2); public final static native boolean ClusteringParameters_int_centroids_get(long jarg1, ClusteringParameters jarg1_); public final static native void ClusteringParameters_update_index_set(long jarg1, ClusteringParameters jarg1_, boolean jarg2); public final static native boolean ClusteringParameters_update_index_get(long jarg1, ClusteringParameters jarg1_); public final static native void ClusteringParameters_frozen_centroids_set(long jarg1, ClusteringParameters jarg1_, boolean jarg2); public final static native boolean ClusteringParameters_frozen_centroids_get(long jarg1, ClusteringParameters jarg1_); public final static native void ClusteringParameters_min_points_per_centroid_set(long jarg1, ClusteringParameters jarg1_, int jarg2); public final static native int ClusteringParameters_min_points_per_centroid_get(long jarg1, ClusteringParameters jarg1_); public final static native void ClusteringParameters_max_points_per_centroid_set(long jarg1, ClusteringParameters jarg1_, int jarg2); public final static native int ClusteringParameters_max_points_per_centroid_get(long jarg1, ClusteringParameters jarg1_); public final static native void ClusteringParameters_seed_set(long jarg1, ClusteringParameters jarg1_, int jarg2); public final static native int ClusteringParameters_seed_get(long jarg1, ClusteringParameters jarg1_); public final static native void ClusteringParameters_decode_block_size_set(long jarg1, ClusteringParameters jarg1_, long jarg2); public final static native long ClusteringParameters_decode_block_size_get(long jarg1, ClusteringParameters jarg1_); public final static native long new_ClusteringParameters(); public final static native void delete_ClusteringParameters(long jarg1); public final static native void ClusteringIterationStats_obj_set(long jarg1, ClusteringIterationStats jarg1_, float jarg2); public final static native float ClusteringIterationStats_obj_get(long jarg1, ClusteringIterationStats jarg1_); public final static native void ClusteringIterationStats_time_set(long jarg1, ClusteringIterationStats jarg1_, double jarg2); public final static native double ClusteringIterationStats_time_get(long jarg1, ClusteringIterationStats jarg1_); public final static native void ClusteringIterationStats_time_search_set(long jarg1, ClusteringIterationStats jarg1_, double jarg2); public final static native double ClusteringIterationStats_time_search_get(long jarg1, ClusteringIterationStats jarg1_); public final static native void ClusteringIterationStats_imbalance_factor_set(long jarg1, ClusteringIterationStats jarg1_, double jarg2); public final static native double ClusteringIterationStats_imbalance_factor_get(long jarg1, ClusteringIterationStats jarg1_); public final static native void ClusteringIterationStats_nsplit_set(long jarg1, ClusteringIterationStats jarg1_, int jarg2); public final static native int ClusteringIterationStats_nsplit_get(long jarg1, ClusteringIterationStats jarg1_); public final static native long new_ClusteringIterationStats(); public final static native void delete_ClusteringIterationStats(long jarg1); public final static native void Clustering_d_set(long jarg1, Clustering jarg1_, long jarg2); public final static native long Clustering_d_get(long jarg1, Clustering jarg1_); public final static native void Clustering_k_set(long jarg1, Clustering jarg1_, long jarg2); public final static native long Clustering_k_get(long jarg1, Clustering jarg1_); public final static native void Clustering_centroids_set(long jarg1, Clustering jarg1_, long jarg2, FloatVector jarg2_); public final static native long Clustering_centroids_get(long jarg1, Clustering jarg1_); public final static native void Clustering_iteration_stats_set(long jarg1, Clustering jarg1_, long jarg2); public final static native long Clustering_iteration_stats_get(long jarg1, Clustering jarg1_); public final static native long new_Clustering__SWIG_0(int jarg1, int jarg2); public final static native long new_Clustering__SWIG_1(int jarg1, int jarg2, long jarg3, ClusteringParameters jarg3_); public final static native void Clustering_train__SWIG_0(long jarg1, Clustering jarg1_, long jarg2, long jarg3, long jarg4, Index jarg4_, long jarg5); public final static native void Clustering_train__SWIG_1(long jarg1, Clustering jarg1_, long jarg2, long jarg3, long jarg4, Index jarg4_); public final static native void Clustering_train_encoded__SWIG_0(long jarg1, Clustering jarg1_, long jarg2, long jarg3, long jarg4, Index jarg4_, long jarg5, Index jarg5_, long jarg6); public final static native void Clustering_train_encoded__SWIG_1(long jarg1, Clustering jarg1_, long jarg2, long jarg3, long jarg4, Index jarg4_, long jarg5, Index jarg5_); public final static native void Clustering_post_process_centroids(long jarg1, Clustering jarg1_); public final static native void delete_Clustering(long jarg1); public final static native long new_Clustering1D__SWIG_0(int jarg1); public final static native long new_Clustering1D__SWIG_1(int jarg1, long jarg2, ClusteringParameters jarg2_); public final static native void Clustering1D_train_exact(long jarg1, Clustering1D jarg1_, long jarg2, long jarg3); public final static native void delete_Clustering1D(long jarg1); public final static native void ProgressiveDimClusteringParameters_progressive_dim_steps_set(long jarg1, ProgressiveDimClusteringParameters jarg1_, int jarg2); public final static native int ProgressiveDimClusteringParameters_progressive_dim_steps_get(long jarg1, ProgressiveDimClusteringParameters jarg1_); public final static native void ProgressiveDimClusteringParameters_apply_pca_set(long jarg1, ProgressiveDimClusteringParameters jarg1_, boolean jarg2); public final static native boolean ProgressiveDimClusteringParameters_apply_pca_get(long jarg1, ProgressiveDimClusteringParameters jarg1_); public final static native long new_ProgressiveDimClusteringParameters(); public final static native void delete_ProgressiveDimClusteringParameters(long jarg1); public final static native void delete_ProgressiveDimIndexFactory(long jarg1); public final static native long new_ProgressiveDimIndexFactory(); public final static native void ProgressiveDimClustering_d_set(long jarg1, ProgressiveDimClustering jarg1_, long jarg2); public final static native long ProgressiveDimClustering_d_get(long jarg1, ProgressiveDimClustering jarg1_); public final static native void ProgressiveDimClustering_k_set(long jarg1, ProgressiveDimClustering jarg1_, long jarg2); public final static native long ProgressiveDimClustering_k_get(long jarg1, ProgressiveDimClustering jarg1_); public final static native void ProgressiveDimClustering_centroids_set(long jarg1, ProgressiveDimClustering jarg1_, long jarg2, FloatVector jarg2_); public final static native long ProgressiveDimClustering_centroids_get(long jarg1, ProgressiveDimClustering jarg1_); public final static native void ProgressiveDimClustering_iteration_stats_set(long jarg1, ProgressiveDimClustering jarg1_, long jarg2); public final static native long ProgressiveDimClustering_iteration_stats_get(long jarg1, ProgressiveDimClustering jarg1_); public final static native long new_ProgressiveDimClustering__SWIG_0(int jarg1, int jarg2); public final static native long new_ProgressiveDimClustering__SWIG_1(int jarg1, int jarg2, long jarg3, ProgressiveDimClusteringParameters jarg3_); public final static native void ProgressiveDimClustering_train(long jarg1, ProgressiveDimClustering jarg1_, long jarg2, long jarg3, long jarg4, ProgressiveDimIndexFactory jarg4_); public final static native void delete_ProgressiveDimClustering(long jarg1); public final static native float kmeans_clustering(long jarg1, long jarg2, long jarg3, long jarg4, long jarg5); public final static native void ProductQuantizer_d_set(long jarg1, ProductQuantizer jarg1_, long jarg2); public final static native long ProductQuantizer_d_get(long jarg1, ProductQuantizer jarg1_); public final static native void ProductQuantizer_M_set(long jarg1, ProductQuantizer jarg1_, long jarg2); public final static native long ProductQuantizer_M_get(long jarg1, ProductQuantizer jarg1_); public final static native void ProductQuantizer_nbits_set(long jarg1, ProductQuantizer jarg1_, long jarg2); public final static native long ProductQuantizer_nbits_get(long jarg1, ProductQuantizer jarg1_); public final static native void ProductQuantizer_dsub_set(long jarg1, ProductQuantizer jarg1_, long jarg2); public final static native long ProductQuantizer_dsub_get(long jarg1, ProductQuantizer jarg1_); public final static native void ProductQuantizer_code_size_set(long jarg1, ProductQuantizer jarg1_, long jarg2); public final static native long ProductQuantizer_code_size_get(long jarg1, ProductQuantizer jarg1_); public final static native void ProductQuantizer_ksub_set(long jarg1, ProductQuantizer jarg1_, long jarg2); public final static native long ProductQuantizer_ksub_get(long jarg1, ProductQuantizer jarg1_); public final static native void ProductQuantizer_verbose_set(long jarg1, ProductQuantizer jarg1_, boolean jarg2); public final static native boolean ProductQuantizer_verbose_get(long jarg1, ProductQuantizer jarg1_); public final static native void ProductQuantizer_train_type_set(long jarg1, ProductQuantizer jarg1_, int jarg2); public final static native int ProductQuantizer_train_type_get(long jarg1, ProductQuantizer jarg1_); public final static native void ProductQuantizer_cp_set(long jarg1, ProductQuantizer jarg1_, long jarg2, ClusteringParameters jarg2_); public final static native long ProductQuantizer_cp_get(long jarg1, ProductQuantizer jarg1_); public final static native void ProductQuantizer_assign_index_set(long jarg1, ProductQuantizer jarg1_, long jarg2, Index jarg2_); public final static native long ProductQuantizer_assign_index_get(long jarg1, ProductQuantizer jarg1_); public final static native void ProductQuantizer_centroids_set(long jarg1, ProductQuantizer jarg1_, long jarg2, FloatVector jarg2_); public final static native long ProductQuantizer_centroids_get(long jarg1, ProductQuantizer jarg1_); public final static native long ProductQuantizer_get_centroids(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3); public final static native void ProductQuantizer_train(long jarg1, ProductQuantizer jarg1_, int jarg2, long jarg3); public final static native long new_ProductQuantizer__SWIG_0(long jarg1, long jarg2, long jarg3); public final static native long new_ProductQuantizer__SWIG_1(); public final static native void ProductQuantizer_set_derived_values(long jarg1, ProductQuantizer jarg1_); public final static native void ProductQuantizer_set_params(long jarg1, ProductQuantizer jarg1_, long jarg2, int jarg3); public final static native void ProductQuantizer_compute_code(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3); public final static native void ProductQuantizer_compute_codes(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3, long jarg4); public final static native void ProductQuantizer_compute_codes_with_assign_index(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3, long jarg4); public final static native void ProductQuantizer_decode__SWIG_0(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3); public final static native void ProductQuantizer_decode__SWIG_1(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3, long jarg4); public final static native void ProductQuantizer_compute_code_from_distance_table(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3); public final static native void ProductQuantizer_compute_distance_table(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3); public final static native void ProductQuantizer_compute_inner_prod_table(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3); public final static native void ProductQuantizer_compute_distance_tables(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3, long jarg4); public final static native void ProductQuantizer_compute_inner_prod_tables(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3, long jarg4); public final static native void ProductQuantizer_search__SWIG_0(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, boolean jarg7); public final static native void ProductQuantizer_search__SWIG_1(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6); public final static native void ProductQuantizer_search_ip__SWIG_0(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, boolean jarg7); public final static native void ProductQuantizer_search_ip__SWIG_1(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6); public final static native void ProductQuantizer_sdc_table_set(long jarg1, ProductQuantizer jarg1_, long jarg2, FloatVector jarg2_); public final static native long ProductQuantizer_sdc_table_get(long jarg1, ProductQuantizer jarg1_); public final static native void ProductQuantizer_compute_sdc_table(long jarg1, ProductQuantizer jarg1_); public final static native void ProductQuantizer_search_sdc__SWIG_0(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, boolean jarg7); public final static native void ProductQuantizer_search_sdc__SWIG_1(long jarg1, ProductQuantizer jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6); public final static native void delete_ProductQuantizer(long jarg1); public final static native void PQEncoderGeneric_code_set(long jarg1, PQEncoderGeneric jarg1_, long jarg2); public final static native long PQEncoderGeneric_code_get(long jarg1, PQEncoderGeneric jarg1_); public final static native void PQEncoderGeneric_offset_set(long jarg1, PQEncoderGeneric jarg1_, short jarg2); public final static native short PQEncoderGeneric_offset_get(long jarg1, PQEncoderGeneric jarg1_); public final static native int PQEncoderGeneric_nbits_get(long jarg1, PQEncoderGeneric jarg1_); public final static native void PQEncoderGeneric_reg_set(long jarg1, PQEncoderGeneric jarg1_, short jarg2); public final static native short PQEncoderGeneric_reg_get(long jarg1, PQEncoderGeneric jarg1_); public final static native long new_PQEncoderGeneric__SWIG_0(long jarg1, int jarg2, short jarg3); public final static native long new_PQEncoderGeneric__SWIG_1(long jarg1, int jarg2); public final static native void PQEncoderGeneric_encode(long jarg1, PQEncoderGeneric jarg1_, long jarg2); public final static native void delete_PQEncoderGeneric(long jarg1); public final static native void PQEncoder8_code_set(long jarg1, PQEncoder8 jarg1_, long jarg2); public final static native long PQEncoder8_code_get(long jarg1, PQEncoder8 jarg1_); public final static native long new_PQEncoder8(long jarg1, int jarg2); public final static native void PQEncoder8_encode(long jarg1, PQEncoder8 jarg1_, long jarg2); public final static native void delete_PQEncoder8(long jarg1); public final static native void PQEncoder16_code_set(long jarg1, PQEncoder16 jarg1_, long jarg2); public final static native long PQEncoder16_code_get(long jarg1, PQEncoder16 jarg1_); public final static native long new_PQEncoder16(long jarg1, int jarg2); public final static native void PQEncoder16_encode(long jarg1, PQEncoder16 jarg1_, long jarg2); public final static native void delete_PQEncoder16(long jarg1); public final static native void PQDecoderGeneric_code_set(long jarg1, PQDecoderGeneric jarg1_, long jarg2); public final static native long PQDecoderGeneric_code_get(long jarg1, PQDecoderGeneric jarg1_); public final static native void PQDecoderGeneric_offset_set(long jarg1, PQDecoderGeneric jarg1_, short jarg2); public final static native short PQDecoderGeneric_offset_get(long jarg1, PQDecoderGeneric jarg1_); public final static native int PQDecoderGeneric_nbits_get(long jarg1, PQDecoderGeneric jarg1_); public final static native long PQDecoderGeneric_mask_get(long jarg1, PQDecoderGeneric jarg1_); public final static native void PQDecoderGeneric_reg_set(long jarg1, PQDecoderGeneric jarg1_, short jarg2); public final static native short PQDecoderGeneric_reg_get(long jarg1, PQDecoderGeneric jarg1_); public final static native long new_PQDecoderGeneric(long jarg1, int jarg2); public final static native long PQDecoderGeneric_decode(long jarg1, PQDecoderGeneric jarg1_); public final static native void delete_PQDecoderGeneric(long jarg1); public final static native int PQDecoder8_nbits_get(); public final static native void PQDecoder8_code_set(long jarg1, PQDecoder8 jarg1_, long jarg2); public final static native long PQDecoder8_code_get(long jarg1, PQDecoder8 jarg1_); public final static native long new_PQDecoder8(long jarg1, int jarg2); public final static native long PQDecoder8_decode(long jarg1, PQDecoder8 jarg1_); public final static native void delete_PQDecoder8(long jarg1); public final static native int PQDecoder16_nbits_get(); public final static native void PQDecoder16_code_set(long jarg1, PQDecoder16 jarg1_, long jarg2); public final static native long PQDecoder16_code_get(long jarg1, PQDecoder16 jarg1_); public final static native long new_PQDecoder16(long jarg1, int jarg2); public final static native long PQDecoder16_decode(long jarg1, PQDecoder16 jarg1_); public final static native void delete_PQDecoder16(long jarg1); public final static native void VectorTransform_d_in_set(long jarg1, VectorTransform jarg1_, int jarg2); public final static native int VectorTransform_d_in_get(long jarg1, VectorTransform jarg1_); public final static native void VectorTransform_d_out_set(long jarg1, VectorTransform jarg1_, int jarg2); public final static native int VectorTransform_d_out_get(long jarg1, VectorTransform jarg1_); public final static native void VectorTransform_is_trained_set(long jarg1, VectorTransform jarg1_, boolean jarg2); public final static native boolean VectorTransform_is_trained_get(long jarg1, VectorTransform jarg1_); public final static native void VectorTransform_train(long jarg1, VectorTransform jarg1_, long jarg2, long jarg3); public final static native long VectorTransform_apply(long jarg1, VectorTransform jarg1_, long jarg2, long jarg3); public final static native void VectorTransform_apply_noalloc(long jarg1, VectorTransform jarg1_, long jarg2, long jarg3, long jarg4); public final static native void VectorTransform_reverse_transform(long jarg1, VectorTransform jarg1_, long jarg2, long jarg3, long jarg4); public final static native void delete_VectorTransform(long jarg1); public final static native void LinearTransform_have_bias_set(long jarg1, LinearTransform jarg1_, boolean jarg2); public final static native boolean LinearTransform_have_bias_get(long jarg1, LinearTransform jarg1_); public final static native void LinearTransform_is_orthonormal_set(long jarg1, LinearTransform jarg1_, boolean jarg2); public final static native boolean LinearTransform_is_orthonormal_get(long jarg1, LinearTransform jarg1_); public final static native void LinearTransform_A_set(long jarg1, LinearTransform jarg1_, long jarg2, FloatVector jarg2_); public final static native long LinearTransform_A_get(long jarg1, LinearTransform jarg1_); public final static native void LinearTransform_b_set(long jarg1, LinearTransform jarg1_, long jarg2, FloatVector jarg2_); public final static native long LinearTransform_b_get(long jarg1, LinearTransform jarg1_); public final static native long new_LinearTransform__SWIG_0(int jarg1, int jarg2, boolean jarg3); public final static native long new_LinearTransform__SWIG_1(int jarg1, int jarg2); public final static native long new_LinearTransform__SWIG_2(int jarg1); public final static native long new_LinearTransform__SWIG_3(); public final static native void LinearTransform_apply_noalloc(long jarg1, LinearTransform jarg1_, long jarg2, long jarg3, long jarg4); public final static native void LinearTransform_transform_transpose(long jarg1, LinearTransform jarg1_, long jarg2, long jarg3, long jarg4); public final static native void LinearTransform_reverse_transform(long jarg1, LinearTransform jarg1_, long jarg2, long jarg3, long jarg4); public final static native void LinearTransform_set_is_orthonormal(long jarg1, LinearTransform jarg1_); public final static native void LinearTransform_verbose_set(long jarg1, LinearTransform jarg1_, boolean jarg2); public final static native boolean LinearTransform_verbose_get(long jarg1, LinearTransform jarg1_); public final static native void LinearTransform_print_if_verbose(long jarg1, LinearTransform jarg1_, String jarg2, long jarg3, DoubleVector jarg3_, int jarg4, int jarg5); public final static native void delete_LinearTransform(long jarg1); public final static native long new_RandomRotationMatrix__SWIG_0(int jarg1, int jarg2); public final static native void RandomRotationMatrix_init(long jarg1, RandomRotationMatrix jarg1_, int jarg2); public final static native void RandomRotationMatrix_train(long jarg1, RandomRotationMatrix jarg1_, long jarg2, long jarg3); public final static native long new_RandomRotationMatrix__SWIG_1(); public final static native void delete_RandomRotationMatrix(long jarg1); public final static native void PCAMatrix_eigen_power_set(long jarg1, PCAMatrix jarg1_, float jarg2); public final static native float PCAMatrix_eigen_power_get(long jarg1, PCAMatrix jarg1_); public final static native void PCAMatrix_epsilon_set(long jarg1, PCAMatrix jarg1_, float jarg2); public final static native float PCAMatrix_epsilon_get(long jarg1, PCAMatrix jarg1_); public final static native void PCAMatrix_random_rotation_set(long jarg1, PCAMatrix jarg1_, boolean jarg2); public final static native boolean PCAMatrix_random_rotation_get(long jarg1, PCAMatrix jarg1_); public final static native void PCAMatrix_max_points_per_d_set(long jarg1, PCAMatrix jarg1_, long jarg2); public final static native long PCAMatrix_max_points_per_d_get(long jarg1, PCAMatrix jarg1_); public final static native void PCAMatrix_balanced_bins_set(long jarg1, PCAMatrix jarg1_, int jarg2); public final static native int PCAMatrix_balanced_bins_get(long jarg1, PCAMatrix jarg1_); public final static native void PCAMatrix_mean_set(long jarg1, PCAMatrix jarg1_, long jarg2, FloatVector jarg2_); public final static native long PCAMatrix_mean_get(long jarg1, PCAMatrix jarg1_); public final static native void PCAMatrix_eigenvalues_set(long jarg1, PCAMatrix jarg1_, long jarg2, FloatVector jarg2_); public final static native long PCAMatrix_eigenvalues_get(long jarg1, PCAMatrix jarg1_); public final static native void PCAMatrix_PCAMat_set(long jarg1, PCAMatrix jarg1_, long jarg2, FloatVector jarg2_); public final static native long PCAMatrix_PCAMat_get(long jarg1, PCAMatrix jarg1_); public final static native long new_PCAMatrix__SWIG_0(int jarg1, int jarg2, float jarg3, boolean jarg4); public final static native long new_PCAMatrix__SWIG_1(int jarg1, int jarg2, float jarg3); public final static native long new_PCAMatrix__SWIG_2(int jarg1, int jarg2); public final static native long new_PCAMatrix__SWIG_3(int jarg1); public final static native long new_PCAMatrix__SWIG_4(); public final static native void PCAMatrix_train(long jarg1, PCAMatrix jarg1_, long jarg2, long jarg3); public final static native void PCAMatrix_copy_from(long jarg1, PCAMatrix jarg1_, long jarg2, PCAMatrix jarg2_); public final static native void PCAMatrix_prepare_Ab(long jarg1, PCAMatrix jarg1_); public final static native void delete_PCAMatrix(long jarg1); public final static native void ITQMatrix_max_iter_set(long jarg1, ITQMatrix jarg1_, int jarg2); public final static native int ITQMatrix_max_iter_get(long jarg1, ITQMatrix jarg1_); public final static native void ITQMatrix_seed_set(long jarg1, ITQMatrix jarg1_, int jarg2); public final static native int ITQMatrix_seed_get(long jarg1, ITQMatrix jarg1_); public final static native void ITQMatrix_init_rotation_set(long jarg1, ITQMatrix jarg1_, long jarg2, DoubleVector jarg2_); public final static native long ITQMatrix_init_rotation_get(long jarg1, ITQMatrix jarg1_); public final static native long new_ITQMatrix__SWIG_0(int jarg1); public final static native long new_ITQMatrix__SWIG_1(); public final static native void ITQMatrix_train(long jarg1, ITQMatrix jarg1_, long jarg2, long jarg3); public final static native void delete_ITQMatrix(long jarg1); public final static native void ITQTransform_mean_set(long jarg1, ITQTransform jarg1_, long jarg2, FloatVector jarg2_); public final static native long ITQTransform_mean_get(long jarg1, ITQTransform jarg1_); public final static native void ITQTransform_do_pca_set(long jarg1, ITQTransform jarg1_, boolean jarg2); public final static native boolean ITQTransform_do_pca_get(long jarg1, ITQTransform jarg1_); public final static native void ITQTransform_itq_set(long jarg1, ITQTransform jarg1_, long jarg2, ITQMatrix jarg2_); public final static native long ITQTransform_itq_get(long jarg1, ITQTransform jarg1_); public final static native void ITQTransform_max_train_per_dim_set(long jarg1, ITQTransform jarg1_, int jarg2); public final static native int ITQTransform_max_train_per_dim_get(long jarg1, ITQTransform jarg1_); public final static native void ITQTransform_pca_then_itq_set(long jarg1, ITQTransform jarg1_, long jarg2, LinearTransform jarg2_); public final static native long ITQTransform_pca_then_itq_get(long jarg1, ITQTransform jarg1_); public final static native long new_ITQTransform__SWIG_0(int jarg1, int jarg2, boolean jarg3); public final static native long new_ITQTransform__SWIG_1(int jarg1, int jarg2); public final static native long new_ITQTransform__SWIG_2(int jarg1); public final static native long new_ITQTransform__SWIG_3(); public final static native void ITQTransform_train(long jarg1, ITQTransform jarg1_, long jarg2, long jarg3); public final static native void ITQTransform_apply_noalloc(long jarg1, ITQTransform jarg1_, long jarg2, long jarg3, long jarg4); public final static native void delete_ITQTransform(long jarg1); public final static native void OPQMatrix_M_set(long jarg1, OPQMatrix jarg1_, int jarg2); public final static native int OPQMatrix_M_get(long jarg1, OPQMatrix jarg1_); public final static native void OPQMatrix_niter_set(long jarg1, OPQMatrix jarg1_, int jarg2); public final static native int OPQMatrix_niter_get(long jarg1, OPQMatrix jarg1_); public final static native void OPQMatrix_niter_pq_set(long jarg1, OPQMatrix jarg1_, int jarg2); public final static native int OPQMatrix_niter_pq_get(long jarg1, OPQMatrix jarg1_); public final static native void OPQMatrix_niter_pq_0_set(long jarg1, OPQMatrix jarg1_, int jarg2); public final static native int OPQMatrix_niter_pq_0_get(long jarg1, OPQMatrix jarg1_); public final static native void OPQMatrix_max_train_points_set(long jarg1, OPQMatrix jarg1_, long jarg2); public final static native long OPQMatrix_max_train_points_get(long jarg1, OPQMatrix jarg1_); public final static native void OPQMatrix_verbose_set(long jarg1, OPQMatrix jarg1_, boolean jarg2); public final static native boolean OPQMatrix_verbose_get(long jarg1, OPQMatrix jarg1_); public final static native void OPQMatrix_pq_set(long jarg1, OPQMatrix jarg1_, long jarg2, ProductQuantizer jarg2_); public final static native long OPQMatrix_pq_get(long jarg1, OPQMatrix jarg1_); public final static native long new_OPQMatrix__SWIG_0(int jarg1, int jarg2, int jarg3); public final static native long new_OPQMatrix__SWIG_1(int jarg1, int jarg2); public final static native long new_OPQMatrix__SWIG_2(int jarg1); public final static native long new_OPQMatrix__SWIG_3(); public final static native void OPQMatrix_train(long jarg1, OPQMatrix jarg1_, long jarg2, long jarg3); public final static native void delete_OPQMatrix(long jarg1); public final static native void RemapDimensionsTransform_map_set(long jarg1, RemapDimensionsTransform jarg1_, long jarg2, IntVector jarg2_); public final static native long RemapDimensionsTransform_map_get(long jarg1, RemapDimensionsTransform jarg1_); public final static native long new_RemapDimensionsTransform__SWIG_0(int jarg1, int jarg2, long jarg3); public final static native long new_RemapDimensionsTransform__SWIG_1(int jarg1, int jarg2, boolean jarg3); public final static native long new_RemapDimensionsTransform__SWIG_2(int jarg1, int jarg2); public final static native void RemapDimensionsTransform_apply_noalloc(long jarg1, RemapDimensionsTransform jarg1_, long jarg2, long jarg3, long jarg4); public final static native void RemapDimensionsTransform_reverse_transform(long jarg1, RemapDimensionsTransform jarg1_, long jarg2, long jarg3, long jarg4); public final static native long new_RemapDimensionsTransform__SWIG_3(); public final static native void delete_RemapDimensionsTransform(long jarg1); public final static native void NormalizationTransform_norm_set(long jarg1, NormalizationTransform jarg1_, float jarg2); public final static native float NormalizationTransform_norm_get(long jarg1, NormalizationTransform jarg1_); public final static native long new_NormalizationTransform__SWIG_0(int jarg1, float jarg2); public final static native long new_NormalizationTransform__SWIG_1(int jarg1); public final static native long new_NormalizationTransform__SWIG_2(); public final static native void NormalizationTransform_apply_noalloc(long jarg1, NormalizationTransform jarg1_, long jarg2, long jarg3, long jarg4); public final static native void NormalizationTransform_reverse_transform(long jarg1, NormalizationTransform jarg1_, long jarg2, long jarg3, long jarg4); public final static native void delete_NormalizationTransform(long jarg1); public final static native void CenteringTransform_mean_set(long jarg1, CenteringTransform jarg1_, long jarg2, FloatVector jarg2_); public final static native long CenteringTransform_mean_get(long jarg1, CenteringTransform jarg1_); public final static native long new_CenteringTransform__SWIG_0(int jarg1); public final static native long new_CenteringTransform__SWIG_1(); public final static native void CenteringTransform_train(long jarg1, CenteringTransform jarg1_, long jarg2, long jarg3); public final static native void CenteringTransform_apply_noalloc(long jarg1, CenteringTransform jarg1_, long jarg2, long jarg3, long jarg4); public final static native void CenteringTransform_reverse_transform(long jarg1, CenteringTransform jarg1_, long jarg2, long jarg3, long jarg4); public final static native void delete_CenteringTransform(long jarg1); public final static native void IndexFlatCodes_code_size_set(long jarg1, IndexFlatCodes jarg1_, long jarg2); public final static native long IndexFlatCodes_code_size_get(long jarg1, IndexFlatCodes jarg1_); public final static native void IndexFlatCodes_codes_set(long jarg1, IndexFlatCodes jarg1_, long jarg2, ByteVector jarg2_); public final static native long IndexFlatCodes_codes_get(long jarg1, IndexFlatCodes jarg1_); public final static native void IndexFlatCodes_add(long jarg1, IndexFlatCodes jarg1_, long jarg2, long jarg3); public final static native void IndexFlatCodes_reset(long jarg1, IndexFlatCodes jarg1_); public final static native void IndexFlatCodes_reconstruct_n(long jarg1, IndexFlatCodes jarg1_, long jarg2, long jarg3, long jarg4); public final static native void IndexFlatCodes_reconstruct(long jarg1, IndexFlatCodes jarg1_, long jarg2, long jarg3); public final static native long IndexFlatCodes_sa_code_size(long jarg1, IndexFlatCodes jarg1_); public final static native long IndexFlatCodes_remove_ids(long jarg1, IndexFlatCodes jarg1_, long jarg2, IDSelector jarg2_); public final static native void delete_IndexFlatCodes(long jarg1); public final static native long new_IndexFlat__SWIG_0(long jarg1, int jarg2); public final static native long new_IndexFlat__SWIG_1(long jarg1); public final static native void IndexFlat_search(long jarg1, IndexFlat jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexFlat_range_search(long jarg1, IndexFlat jarg1_, long jarg2, long jarg3, float jarg4, long jarg5, RangeSearchResult jarg5_); public final static native void IndexFlat_reconstruct(long jarg1, IndexFlat jarg1_, long jarg2, long jarg3); public final static native void IndexFlat_compute_distance_subset(long jarg1, IndexFlat jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native long IndexFlat_get_xb__SWIG_0(long jarg1, IndexFlat jarg1_); public final static native long new_IndexFlat__SWIG_2(); public final static native long IndexFlat_get_distance_computer(long jarg1, IndexFlat jarg1_); public final static native void IndexFlat_sa_encode(long jarg1, IndexFlat jarg1_, long jarg2, long jarg3, long jarg4); public final static native void IndexFlat_sa_decode(long jarg1, IndexFlat jarg1_, long jarg2, long jarg3, long jarg4); public final static native void delete_IndexFlat(long jarg1); public final static native long new_IndexFlatIP__SWIG_0(long jarg1); public final static native long new_IndexFlatIP__SWIG_1(); public final static native void delete_IndexFlatIP(long jarg1); public final static native long new_IndexFlatL2__SWIG_0(long jarg1); public final static native long new_IndexFlatL2__SWIG_1(); public final static native void delete_IndexFlatL2(long jarg1); public final static native void IndexFlat1D_continuous_update_set(long jarg1, IndexFlat1D jarg1_, boolean jarg2); public final static native boolean IndexFlat1D_continuous_update_get(long jarg1, IndexFlat1D jarg1_); public final static native void IndexFlat1D_perm_set(long jarg1, IndexFlat1D jarg1_, long jarg2); public final static native long IndexFlat1D_perm_get(long jarg1, IndexFlat1D jarg1_); public final static native long new_IndexFlat1D__SWIG_0(boolean jarg1); public final static native long new_IndexFlat1D__SWIG_1(); public final static native void IndexFlat1D_update_permutation(long jarg1, IndexFlat1D jarg1_); public final static native void IndexFlat1D_add(long jarg1, IndexFlat1D jarg1_, long jarg2, long jarg3); public final static native void IndexFlat1D_reset(long jarg1, IndexFlat1D jarg1_); public final static native void IndexFlat1D_search(long jarg1, IndexFlat1D jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void delete_IndexFlat1D(long jarg1); public final static native void IndexLSH_nbits_set(long jarg1, IndexLSH jarg1_, int jarg2); public final static native int IndexLSH_nbits_get(long jarg1, IndexLSH jarg1_); public final static native void IndexLSH_rotate_data_set(long jarg1, IndexLSH jarg1_, boolean jarg2); public final static native boolean IndexLSH_rotate_data_get(long jarg1, IndexLSH jarg1_); public final static native void IndexLSH_train_thresholds_set(long jarg1, IndexLSH jarg1_, boolean jarg2); public final static native boolean IndexLSH_train_thresholds_get(long jarg1, IndexLSH jarg1_); public final static native void IndexLSH_rrot_set(long jarg1, IndexLSH jarg1_, long jarg2, RandomRotationMatrix jarg2_); public final static native long IndexLSH_rrot_get(long jarg1, IndexLSH jarg1_); public final static native void IndexLSH_thresholds_set(long jarg1, IndexLSH jarg1_, long jarg2, FloatVector jarg2_); public final static native long IndexLSH_thresholds_get(long jarg1, IndexLSH jarg1_); public final static native long new_IndexLSH__SWIG_0(long jarg1, int jarg2, boolean jarg3, boolean jarg4); public final static native long new_IndexLSH__SWIG_1(long jarg1, int jarg2, boolean jarg3); public final static native long new_IndexLSH__SWIG_2(long jarg1, int jarg2); public final static native long IndexLSH_apply_preprocess(long jarg1, IndexLSH jarg1_, long jarg2, long jarg3); public final static native void IndexLSH_train(long jarg1, IndexLSH jarg1_, long jarg2, long jarg3); public final static native void IndexLSH_search(long jarg1, IndexLSH jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexLSH_transfer_thresholds(long jarg1, IndexLSH jarg1_, long jarg2, LinearTransform jarg2_); public final static native void delete_IndexLSH(long jarg1); public final static native long new_IndexLSH__SWIG_3(); public final static native void IndexLSH_sa_encode(long jarg1, IndexLSH jarg1_, long jarg2, long jarg3, long jarg4); public final static native void IndexLSH_sa_decode(long jarg1, IndexLSH jarg1_, long jarg2, long jarg3, long jarg4); public final static native void SimulatedAnnealingParameters_init_temperature_set(long jarg1, SimulatedAnnealingParameters jarg1_, double jarg2); public final static native double SimulatedAnnealingParameters_init_temperature_get(long jarg1, SimulatedAnnealingParameters jarg1_); public final static native void SimulatedAnnealingParameters_temperature_decay_set(long jarg1, SimulatedAnnealingParameters jarg1_, double jarg2); public final static native double SimulatedAnnealingParameters_temperature_decay_get(long jarg1, SimulatedAnnealingParameters jarg1_); public final static native void SimulatedAnnealingParameters_n_iter_set(long jarg1, SimulatedAnnealingParameters jarg1_, int jarg2); public final static native int SimulatedAnnealingParameters_n_iter_get(long jarg1, SimulatedAnnealingParameters jarg1_); public final static native void SimulatedAnnealingParameters_n_redo_set(long jarg1, SimulatedAnnealingParameters jarg1_, int jarg2); public final static native int SimulatedAnnealingParameters_n_redo_get(long jarg1, SimulatedAnnealingParameters jarg1_); public final static native void SimulatedAnnealingParameters_seed_set(long jarg1, SimulatedAnnealingParameters jarg1_, int jarg2); public final static native int SimulatedAnnealingParameters_seed_get(long jarg1, SimulatedAnnealingParameters jarg1_); public final static native void SimulatedAnnealingParameters_verbose_set(long jarg1, SimulatedAnnealingParameters jarg1_, int jarg2); public final static native int SimulatedAnnealingParameters_verbose_get(long jarg1, SimulatedAnnealingParameters jarg1_); public final static native void SimulatedAnnealingParameters_only_bit_flips_set(long jarg1, SimulatedAnnealingParameters jarg1_, boolean jarg2); public final static native boolean SimulatedAnnealingParameters_only_bit_flips_get(long jarg1, SimulatedAnnealingParameters jarg1_); public final static native void SimulatedAnnealingParameters_init_random_set(long jarg1, SimulatedAnnealingParameters jarg1_, boolean jarg2); public final static native boolean SimulatedAnnealingParameters_init_random_get(long jarg1, SimulatedAnnealingParameters jarg1_); public final static native long new_SimulatedAnnealingParameters(); public final static native void delete_SimulatedAnnealingParameters(long jarg1); public final static native void PermutationObjective_n_set(long jarg1, PermutationObjective jarg1_, int jarg2); public final static native int PermutationObjective_n_get(long jarg1, PermutationObjective jarg1_); public final static native double PermutationObjective_compute_cost(long jarg1, PermutationObjective jarg1_, long jarg2); public final static native double PermutationObjective_cost_update(long jarg1, PermutationObjective jarg1_, long jarg2, int jarg3, int jarg4); public final static native void delete_PermutationObjective(long jarg1); public final static native void ReproduceDistancesObjective_dis_weight_factor_set(long jarg1, ReproduceDistancesObjective jarg1_, double jarg2); public final static native double ReproduceDistancesObjective_dis_weight_factor_get(long jarg1, ReproduceDistancesObjective jarg1_); public final static native double ReproduceDistancesObjective_sqr(double jarg1); public final static native double ReproduceDistancesObjective_dis_weight(long jarg1, ReproduceDistancesObjective jarg1_, double jarg2); public final static native void ReproduceDistancesObjective_source_dis_set(long jarg1, ReproduceDistancesObjective jarg1_, long jarg2, DoubleVector jarg2_); public final static native long ReproduceDistancesObjective_source_dis_get(long jarg1, ReproduceDistancesObjective jarg1_); public final static native void ReproduceDistancesObjective_target_dis_set(long jarg1, ReproduceDistancesObjective jarg1_, long jarg2); public final static native long ReproduceDistancesObjective_target_dis_get(long jarg1, ReproduceDistancesObjective jarg1_); public final static native void ReproduceDistancesObjective_weights_set(long jarg1, ReproduceDistancesObjective jarg1_, long jarg2, DoubleVector jarg2_); public final static native long ReproduceDistancesObjective_weights_get(long jarg1, ReproduceDistancesObjective jarg1_); public final static native double ReproduceDistancesObjective_get_source_dis(long jarg1, ReproduceDistancesObjective jarg1_, int jarg2, int jarg3); public final static native double ReproduceDistancesObjective_compute_cost(long jarg1, ReproduceDistancesObjective jarg1_, long jarg2); public final static native double ReproduceDistancesObjective_cost_update(long jarg1, ReproduceDistancesObjective jarg1_, long jarg2, int jarg3, int jarg4); public final static native long new_ReproduceDistancesObjective(int jarg1, long jarg2, long jarg3, double jarg4); public final static native void ReproduceDistancesObjective_compute_mean_stdev(long jarg1, long jarg2, long jarg3, long jarg4); public final static native void ReproduceDistancesObjective_set_affine_target_dis(long jarg1, ReproduceDistancesObjective jarg1_, long jarg2); public final static native void delete_ReproduceDistancesObjective(long jarg1); public final static native void SimulatedAnnealingOptimizer_obj_set(long jarg1, SimulatedAnnealingOptimizer jarg1_, long jarg2, PermutationObjective jarg2_); public final static native long SimulatedAnnealingOptimizer_obj_get(long jarg1, SimulatedAnnealingOptimizer jarg1_); public final static native void SimulatedAnnealingOptimizer_n_set(long jarg1, SimulatedAnnealingOptimizer jarg1_, int jarg2); public final static native int SimulatedAnnealingOptimizer_n_get(long jarg1, SimulatedAnnealingOptimizer jarg1_); public final static native void SimulatedAnnealingOptimizer_logfile_set(long jarg1, SimulatedAnnealingOptimizer jarg1_, long jarg2); public final static native long SimulatedAnnealingOptimizer_logfile_get(long jarg1, SimulatedAnnealingOptimizer jarg1_); public final static native long new_SimulatedAnnealingOptimizer(long jarg1, PermutationObjective jarg1_, long jarg2, SimulatedAnnealingParameters jarg2_); public final static native void SimulatedAnnealingOptimizer_rnd_set(long jarg1, SimulatedAnnealingOptimizer jarg1_, long jarg2); public final static native long SimulatedAnnealingOptimizer_rnd_get(long jarg1, SimulatedAnnealingOptimizer jarg1_); public final static native void SimulatedAnnealingOptimizer_init_cost_set(long jarg1, SimulatedAnnealingOptimizer jarg1_, double jarg2); public final static native double SimulatedAnnealingOptimizer_init_cost_get(long jarg1, SimulatedAnnealingOptimizer jarg1_); public final static native double SimulatedAnnealingOptimizer_optimize(long jarg1, SimulatedAnnealingOptimizer jarg1_, long jarg2); public final static native double SimulatedAnnealingOptimizer_run_optimization(long jarg1, SimulatedAnnealingOptimizer jarg1_, long jarg2); public final static native void delete_SimulatedAnnealingOptimizer(long jarg1); public final static native void PolysemousTraining_optimization_type_set(long jarg1, PolysemousTraining jarg1_, int jarg2); public final static native int PolysemousTraining_optimization_type_get(long jarg1, PolysemousTraining jarg1_); public final static native void PolysemousTraining_ntrain_permutation_set(long jarg1, PolysemousTraining jarg1_, int jarg2); public final static native int PolysemousTraining_ntrain_permutation_get(long jarg1, PolysemousTraining jarg1_); public final static native void PolysemousTraining_dis_weight_factor_set(long jarg1, PolysemousTraining jarg1_, double jarg2); public final static native double PolysemousTraining_dis_weight_factor_get(long jarg1, PolysemousTraining jarg1_); public final static native void PolysemousTraining_max_memory_set(long jarg1, PolysemousTraining jarg1_, long jarg2); public final static native long PolysemousTraining_max_memory_get(long jarg1, PolysemousTraining jarg1_); public final static native void PolysemousTraining_log_pattern_set(long jarg1, PolysemousTraining jarg1_, String jarg2); public final static native String PolysemousTraining_log_pattern_get(long jarg1, PolysemousTraining jarg1_); public final static native long new_PolysemousTraining(); public final static native void PolysemousTraining_optimize_pq_for_hamming(long jarg1, PolysemousTraining jarg1_, long jarg2, ProductQuantizer jarg2_, long jarg3, long jarg4); public final static native void PolysemousTraining_optimize_ranking(long jarg1, PolysemousTraining jarg1_, long jarg2, ProductQuantizer jarg2_, long jarg3, long jarg4); public final static native void PolysemousTraining_optimize_reproduce_distances(long jarg1, PolysemousTraining jarg1_, long jarg2, ProductQuantizer jarg2_); public final static native long PolysemousTraining_memory_usage_per_thread(long jarg1, PolysemousTraining jarg1_, long jarg2, ProductQuantizer jarg2_); public final static native void delete_PolysemousTraining(long jarg1); public final static native void IndexPQ_pq_set(long jarg1, IndexPQ jarg1_, long jarg2, ProductQuantizer jarg2_); public final static native long IndexPQ_pq_get(long jarg1, IndexPQ jarg1_); public final static native long new_IndexPQ__SWIG_0(int jarg1, long jarg2, long jarg3, int jarg4); public final static native long new_IndexPQ__SWIG_1(int jarg1, long jarg2, long jarg3); public final static native long new_IndexPQ__SWIG_2(); public final static native void IndexPQ_train(long jarg1, IndexPQ jarg1_, long jarg2, long jarg3); public final static native void IndexPQ_search(long jarg1, IndexPQ jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexPQ_sa_encode(long jarg1, IndexPQ jarg1_, long jarg2, long jarg3, long jarg4); public final static native void IndexPQ_sa_decode(long jarg1, IndexPQ jarg1_, long jarg2, long jarg3, long jarg4); public final static native long IndexPQ_get_distance_computer(long jarg1, IndexPQ jarg1_); public final static native void IndexPQ_do_polysemous_training_set(long jarg1, IndexPQ jarg1_, boolean jarg2); public final static native boolean IndexPQ_do_polysemous_training_get(long jarg1, IndexPQ jarg1_); public final static native void IndexPQ_polysemous_training_set(long jarg1, IndexPQ jarg1_, long jarg2, PolysemousTraining jarg2_); public final static native long IndexPQ_polysemous_training_get(long jarg1, IndexPQ jarg1_); public final static native void IndexPQ_search_type_set(long jarg1, IndexPQ jarg1_, int jarg2); public final static native int IndexPQ_search_type_get(long jarg1, IndexPQ jarg1_); public final static native void IndexPQ_encode_signs_set(long jarg1, IndexPQ jarg1_, boolean jarg2); public final static native boolean IndexPQ_encode_signs_get(long jarg1, IndexPQ jarg1_); public final static native void IndexPQ_polysemous_ht_set(long jarg1, IndexPQ jarg1_, int jarg2); public final static native int IndexPQ_polysemous_ht_get(long jarg1, IndexPQ jarg1_); public final static native void IndexPQ_search_core_polysemous(long jarg1, IndexPQ jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexPQ_hamming_distance_histogram(long jarg1, IndexPQ jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexPQ_hamming_distance_table(long jarg1, IndexPQ jarg1_, long jarg2, long jarg3, long jarg4); public final static native void delete_IndexPQ(long jarg1); public final static native void IndexPQStats_nq_set(long jarg1, IndexPQStats jarg1_, long jarg2); public final static native long IndexPQStats_nq_get(long jarg1, IndexPQStats jarg1_); public final static native void IndexPQStats_ncode_set(long jarg1, IndexPQStats jarg1_, long jarg2); public final static native long IndexPQStats_ncode_get(long jarg1, IndexPQStats jarg1_); public final static native void IndexPQStats_n_hamming_pass_set(long jarg1, IndexPQStats jarg1_, long jarg2); public final static native long IndexPQStats_n_hamming_pass_get(long jarg1, IndexPQStats jarg1_); public final static native long new_IndexPQStats(); public final static native void IndexPQStats_reset(long jarg1, IndexPQStats jarg1_); public final static native void delete_IndexPQStats(long jarg1); public final static native void indexPQ_stats_set(long jarg1, IndexPQStats jarg1_); public final static native long indexPQ_stats_get(); public final static native void MultiIndexQuantizer_pq_set(long jarg1, MultiIndexQuantizer jarg1_, long jarg2, ProductQuantizer jarg2_); public final static native long MultiIndexQuantizer_pq_get(long jarg1, MultiIndexQuantizer jarg1_); public final static native long new_MultiIndexQuantizer__SWIG_0(int jarg1, long jarg2, long jarg3); public final static native void MultiIndexQuantizer_train(long jarg1, MultiIndexQuantizer jarg1_, long jarg2, long jarg3); public final static native void MultiIndexQuantizer_search(long jarg1, MultiIndexQuantizer jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void MultiIndexQuantizer_add(long jarg1, MultiIndexQuantizer jarg1_, long jarg2, long jarg3); public final static native void MultiIndexQuantizer_reset(long jarg1, MultiIndexQuantizer jarg1_); public final static native long new_MultiIndexQuantizer__SWIG_1(); public final static native void MultiIndexQuantizer_reconstruct(long jarg1, MultiIndexQuantizer jarg1_, long jarg2, long jarg3); public final static native void delete_MultiIndexQuantizer(long jarg1); public final static native void MultiIndexQuantizer2_assign_indexes_set(long jarg1, MultiIndexQuantizer2 jarg1_, long jarg2); public final static native long MultiIndexQuantizer2_assign_indexes_get(long jarg1, MultiIndexQuantizer2 jarg1_); public final static native void MultiIndexQuantizer2_own_fields_set(long jarg1, MultiIndexQuantizer2 jarg1_, boolean jarg2); public final static native boolean MultiIndexQuantizer2_own_fields_get(long jarg1, MultiIndexQuantizer2 jarg1_); public final static native long new_MultiIndexQuantizer2__SWIG_0(int jarg1, long jarg2, long jarg3, long jarg4); public final static native long new_MultiIndexQuantizer2__SWIG_1(int jarg1, long jarg2, long jarg3, Index jarg3_, long jarg4, Index jarg4_); public final static native void MultiIndexQuantizer2_train(long jarg1, MultiIndexQuantizer2 jarg1_, long jarg2, long jarg3); public final static native void MultiIndexQuantizer2_search(long jarg1, MultiIndexQuantizer2 jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void delete_MultiIndexQuantizer2(long jarg1); public final static native void InvertedLists_nlist_set(long jarg1, InvertedLists jarg1_, long jarg2); public final static native long InvertedLists_nlist_get(long jarg1, InvertedLists jarg1_); public final static native void InvertedLists_code_size_set(long jarg1, InvertedLists jarg1_, long jarg2); public final static native long InvertedLists_code_size_get(long jarg1, InvertedLists jarg1_); public final static native long InvertedLists_INVALID_CODE_SIZE_get(); public final static native long InvertedLists_list_size(long jarg1, InvertedLists jarg1_, long jarg2); public final static native long InvertedLists_get_codes(long jarg1, InvertedLists jarg1_, long jarg2); public final static native long InvertedLists_get_ids(long jarg1, InvertedLists jarg1_, long jarg2); public final static native void InvertedLists_release_codes(long jarg1, InvertedLists jarg1_, long jarg2, long jarg3); public final static native void InvertedLists_release_ids(long jarg1, InvertedLists jarg1_, long jarg2, long jarg3, LongVector jarg3_); public final static native long InvertedLists_get_single_id(long jarg1, InvertedLists jarg1_, long jarg2, long jarg3); public final static native long InvertedLists_get_single_code(long jarg1, InvertedLists jarg1_, long jarg2, long jarg3); public final static native void InvertedLists_prefetch_lists(long jarg1, InvertedLists jarg1_, long jarg2, LongVector jarg2_, int jarg3); public final static native long InvertedLists_add_entry(long jarg1, InvertedLists jarg1_, long jarg2, long jarg3, long jarg4); public final static native long InvertedLists_add_entries(long jarg1, InvertedLists jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5); public final static native void InvertedLists_update_entry(long jarg1, InvertedLists jarg1_, long jarg2, long jarg3, long jarg4, long jarg5); public final static native void InvertedLists_update_entries(long jarg1, InvertedLists jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, LongVector jarg5_, long jarg6); public final static native void InvertedLists_resize(long jarg1, InvertedLists jarg1_, long jarg2, long jarg3); public final static native void InvertedLists_reset(long jarg1, InvertedLists jarg1_); public final static native void InvertedLists_merge_from(long jarg1, InvertedLists jarg1_, long jarg2, InvertedLists jarg2_, long jarg3); public final static native void delete_InvertedLists(long jarg1); public final static native double InvertedLists_imbalance_factor(long jarg1, InvertedLists jarg1_); public final static native void InvertedLists_print_stats(long jarg1, InvertedLists jarg1_); public final static native long InvertedLists_compute_ntotal(long jarg1, InvertedLists jarg1_); public final static native void InvertedLists_ScopedIds_il_set(long jarg1, InvertedLists.ScopedIds jarg1_, long jarg2, InvertedLists jarg2_); public final static native long InvertedLists_ScopedIds_il_get(long jarg1, InvertedLists.ScopedIds jarg1_); public final static native void InvertedLists_ScopedIds_ids_set(long jarg1, InvertedLists.ScopedIds jarg1_, long jarg2, LongVector jarg2_); public final static native long InvertedLists_ScopedIds_ids_get(long jarg1, InvertedLists.ScopedIds jarg1_); public final static native void InvertedLists_ScopedIds_list_no_set(long jarg1, InvertedLists.ScopedIds jarg1_, long jarg2); public final static native long InvertedLists_ScopedIds_list_no_get(long jarg1, InvertedLists.ScopedIds jarg1_); public final static native long new_InvertedLists_ScopedIds(long jarg1, InvertedLists jarg1_, long jarg2); public final static native long InvertedLists_ScopedIds_get(long jarg1, InvertedLists.ScopedIds jarg1_); public final static native void delete_InvertedLists_ScopedIds(long jarg1); public final static native void InvertedLists_ScopedCodes_il_set(long jarg1, InvertedLists.ScopedCodes jarg1_, long jarg2, InvertedLists jarg2_); public final static native long InvertedLists_ScopedCodes_il_get(long jarg1, InvertedLists.ScopedCodes jarg1_); public final static native void InvertedLists_ScopedCodes_codes_set(long jarg1, InvertedLists.ScopedCodes jarg1_, long jarg2); public final static native long InvertedLists_ScopedCodes_codes_get(long jarg1, InvertedLists.ScopedCodes jarg1_); public final static native void InvertedLists_ScopedCodes_list_no_set(long jarg1, InvertedLists.ScopedCodes jarg1_, long jarg2); public final static native long InvertedLists_ScopedCodes_list_no_get(long jarg1, InvertedLists.ScopedCodes jarg1_); public final static native long new_InvertedLists_ScopedCodes__SWIG_0(long jarg1, InvertedLists jarg1_, long jarg2); public final static native long new_InvertedLists_ScopedCodes__SWIG_1(long jarg1, InvertedLists jarg1_, long jarg2, long jarg3); public final static native long InvertedLists_ScopedCodes_get(long jarg1, InvertedLists.ScopedCodes jarg1_); public final static native void delete_InvertedLists_ScopedCodes(long jarg1); public final static native void ArrayInvertedLists_codes_set(long jarg1, ArrayInvertedLists jarg1_, long jarg2, ByteVectorVector jarg2_); public final static native long ArrayInvertedLists_codes_get(long jarg1, ArrayInvertedLists jarg1_); public final static native void ArrayInvertedLists_ids_set(long jarg1, ArrayInvertedLists jarg1_, long jarg2); public final static native long ArrayInvertedLists_ids_get(long jarg1, ArrayInvertedLists jarg1_); public final static native long new_ArrayInvertedLists(long jarg1, long jarg2); public final static native long ArrayInvertedLists_list_size(long jarg1, ArrayInvertedLists jarg1_, long jarg2); public final static native long ArrayInvertedLists_get_codes(long jarg1, ArrayInvertedLists jarg1_, long jarg2); public final static native long ArrayInvertedLists_get_ids(long jarg1, ArrayInvertedLists jarg1_, long jarg2); public final static native long ArrayInvertedLists_add_entries(long jarg1, ArrayInvertedLists jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5); public final static native void ArrayInvertedLists_update_entries(long jarg1, ArrayInvertedLists jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, LongVector jarg5_, long jarg6); public final static native void ArrayInvertedLists_resize(long jarg1, ArrayInvertedLists jarg1_, long jarg2, long jarg3); public final static native void delete_ArrayInvertedLists(long jarg1); public final static native long ReadOnlyInvertedLists_add_entries(long jarg1, ReadOnlyInvertedLists jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5); public final static native void ReadOnlyInvertedLists_update_entries(long jarg1, ReadOnlyInvertedLists jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, LongVector jarg5_, long jarg6); public final static native void ReadOnlyInvertedLists_resize(long jarg1, ReadOnlyInvertedLists jarg1_, long jarg2, long jarg3); public final static native void delete_ReadOnlyInvertedLists(long jarg1); public final static native void HStackInvertedLists_ils_set(long jarg1, HStackInvertedLists jarg1_, long jarg2); public final static native long HStackInvertedLists_ils_get(long jarg1, HStackInvertedLists jarg1_); public final static native long new_HStackInvertedLists(int jarg1, long jarg2); public final static native long HStackInvertedLists_list_size(long jarg1, HStackInvertedLists jarg1_, long jarg2); public final static native long HStackInvertedLists_get_codes(long jarg1, HStackInvertedLists jarg1_, long jarg2); public final static native long HStackInvertedLists_get_ids(long jarg1, HStackInvertedLists jarg1_, long jarg2); public final static native void HStackInvertedLists_prefetch_lists(long jarg1, HStackInvertedLists jarg1_, long jarg2, LongVector jarg2_, int jarg3); public final static native void HStackInvertedLists_release_codes(long jarg1, HStackInvertedLists jarg1_, long jarg2, long jarg3); public final static native void HStackInvertedLists_release_ids(long jarg1, HStackInvertedLists jarg1_, long jarg2, long jarg3, LongVector jarg3_); public final static native long HStackInvertedLists_get_single_id(long jarg1, HStackInvertedLists jarg1_, long jarg2, long jarg3); public final static native long HStackInvertedLists_get_single_code(long jarg1, HStackInvertedLists jarg1_, long jarg2, long jarg3); public final static native void delete_HStackInvertedLists(long jarg1); public final static native void SliceInvertedLists_il_set(long jarg1, SliceInvertedLists jarg1_, long jarg2, InvertedLists jarg2_); public final static native long SliceInvertedLists_il_get(long jarg1, SliceInvertedLists jarg1_); public final static native void SliceInvertedLists_i0_set(long jarg1, SliceInvertedLists jarg1_, long jarg2); public final static native long SliceInvertedLists_i0_get(long jarg1, SliceInvertedLists jarg1_); public final static native void SliceInvertedLists_i1_set(long jarg1, SliceInvertedLists jarg1_, long jarg2); public final static native long SliceInvertedLists_i1_get(long jarg1, SliceInvertedLists jarg1_); public final static native long new_SliceInvertedLists(long jarg1, InvertedLists jarg1_, long jarg2, long jarg3); public final static native long SliceInvertedLists_list_size(long jarg1, SliceInvertedLists jarg1_, long jarg2); public final static native long SliceInvertedLists_get_codes(long jarg1, SliceInvertedLists jarg1_, long jarg2); public final static native long SliceInvertedLists_get_ids(long jarg1, SliceInvertedLists jarg1_, long jarg2); public final static native void SliceInvertedLists_release_codes(long jarg1, SliceInvertedLists jarg1_, long jarg2, long jarg3); public final static native void SliceInvertedLists_release_ids(long jarg1, SliceInvertedLists jarg1_, long jarg2, long jarg3, LongVector jarg3_); public final static native long SliceInvertedLists_get_single_id(long jarg1, SliceInvertedLists jarg1_, long jarg2, long jarg3); public final static native long SliceInvertedLists_get_single_code(long jarg1, SliceInvertedLists jarg1_, long jarg2, long jarg3); public final static native void SliceInvertedLists_prefetch_lists(long jarg1, SliceInvertedLists jarg1_, long jarg2, LongVector jarg2_, int jarg3); public final static native void delete_SliceInvertedLists(long jarg1); public final static native void VStackInvertedLists_ils_set(long jarg1, VStackInvertedLists jarg1_, long jarg2); public final static native long VStackInvertedLists_ils_get(long jarg1, VStackInvertedLists jarg1_); public final static native void VStackInvertedLists_cumsz_set(long jarg1, VStackInvertedLists jarg1_, long jarg2); public final static native long VStackInvertedLists_cumsz_get(long jarg1, VStackInvertedLists jarg1_); public final static native long new_VStackInvertedLists(int jarg1, long jarg2); public final static native long VStackInvertedLists_list_size(long jarg1, VStackInvertedLists jarg1_, long jarg2); public final static native long VStackInvertedLists_get_codes(long jarg1, VStackInvertedLists jarg1_, long jarg2); public final static native long VStackInvertedLists_get_ids(long jarg1, VStackInvertedLists jarg1_, long jarg2); public final static native void VStackInvertedLists_release_codes(long jarg1, VStackInvertedLists jarg1_, long jarg2, long jarg3); public final static native void VStackInvertedLists_release_ids(long jarg1, VStackInvertedLists jarg1_, long jarg2, long jarg3, LongVector jarg3_); public final static native long VStackInvertedLists_get_single_id(long jarg1, VStackInvertedLists jarg1_, long jarg2, long jarg3); public final static native long VStackInvertedLists_get_single_code(long jarg1, VStackInvertedLists jarg1_, long jarg2, long jarg3); public final static native void VStackInvertedLists_prefetch_lists(long jarg1, VStackInvertedLists jarg1_, long jarg2, LongVector jarg2_, int jarg3); public final static native void delete_VStackInvertedLists(long jarg1); public final static native void MaskedInvertedLists_il0_set(long jarg1, MaskedInvertedLists jarg1_, long jarg2, InvertedLists jarg2_); public final static native long MaskedInvertedLists_il0_get(long jarg1, MaskedInvertedLists jarg1_); public final static native void MaskedInvertedLists_il1_set(long jarg1, MaskedInvertedLists jarg1_, long jarg2, InvertedLists jarg2_); public final static native long MaskedInvertedLists_il1_get(long jarg1, MaskedInvertedLists jarg1_); public final static native long new_MaskedInvertedLists(long jarg1, InvertedLists jarg1_, long jarg2, InvertedLists jarg2_); public final static native long MaskedInvertedLists_list_size(long jarg1, MaskedInvertedLists jarg1_, long jarg2); public final static native long MaskedInvertedLists_get_codes(long jarg1, MaskedInvertedLists jarg1_, long jarg2); public final static native long MaskedInvertedLists_get_ids(long jarg1, MaskedInvertedLists jarg1_, long jarg2); public final static native void MaskedInvertedLists_release_codes(long jarg1, MaskedInvertedLists jarg1_, long jarg2, long jarg3); public final static native void MaskedInvertedLists_release_ids(long jarg1, MaskedInvertedLists jarg1_, long jarg2, long jarg3, LongVector jarg3_); public final static native long MaskedInvertedLists_get_single_id(long jarg1, MaskedInvertedLists jarg1_, long jarg2, long jarg3); public final static native long MaskedInvertedLists_get_single_code(long jarg1, MaskedInvertedLists jarg1_, long jarg2, long jarg3); public final static native void MaskedInvertedLists_prefetch_lists(long jarg1, MaskedInvertedLists jarg1_, long jarg2, LongVector jarg2_, int jarg3); public final static native void delete_MaskedInvertedLists(long jarg1); public final static native void StopWordsInvertedLists_il0_set(long jarg1, StopWordsInvertedLists jarg1_, long jarg2, InvertedLists jarg2_); public final static native long StopWordsInvertedLists_il0_get(long jarg1, StopWordsInvertedLists jarg1_); public final static native void StopWordsInvertedLists_maxsize_set(long jarg1, StopWordsInvertedLists jarg1_, long jarg2); public final static native long StopWordsInvertedLists_maxsize_get(long jarg1, StopWordsInvertedLists jarg1_); public final static native long new_StopWordsInvertedLists(long jarg1, InvertedLists jarg1_, long jarg2); public final static native long StopWordsInvertedLists_list_size(long jarg1, StopWordsInvertedLists jarg1_, long jarg2); public final static native long StopWordsInvertedLists_get_codes(long jarg1, StopWordsInvertedLists jarg1_, long jarg2); public final static native long StopWordsInvertedLists_get_ids(long jarg1, StopWordsInvertedLists jarg1_, long jarg2); public final static native void StopWordsInvertedLists_release_codes(long jarg1, StopWordsInvertedLists jarg1_, long jarg2, long jarg3); public final static native void StopWordsInvertedLists_release_ids(long jarg1, StopWordsInvertedLists jarg1_, long jarg2, long jarg3, LongVector jarg3_); public final static native long StopWordsInvertedLists_get_single_id(long jarg1, StopWordsInvertedLists jarg1_, long jarg2, long jarg3); public final static native long StopWordsInvertedLists_get_single_code(long jarg1, StopWordsInvertedLists jarg1_, long jarg2, long jarg3); public final static native void StopWordsInvertedLists_prefetch_lists(long jarg1, StopWordsInvertedLists jarg1_, long jarg2, LongVector jarg2_, int jarg3); public final static native void delete_StopWordsInvertedLists(long jarg1); public final static native void Level1Quantizer_quantizer_set(long jarg1, Level1Quantizer jarg1_, long jarg2, Index jarg2_); public final static native long Level1Quantizer_quantizer_get(long jarg1, Level1Quantizer jarg1_); public final static native void Level1Quantizer_nlist_set(long jarg1, Level1Quantizer jarg1_, long jarg2); public final static native long Level1Quantizer_nlist_get(long jarg1, Level1Quantizer jarg1_); public final static native void Level1Quantizer_quantizer_trains_alone_set(long jarg1, Level1Quantizer jarg1_, char jarg2); public final static native char Level1Quantizer_quantizer_trains_alone_get(long jarg1, Level1Quantizer jarg1_); public final static native void Level1Quantizer_own_fields_set(long jarg1, Level1Quantizer jarg1_, boolean jarg2); public final static native boolean Level1Quantizer_own_fields_get(long jarg1, Level1Quantizer jarg1_); public final static native void Level1Quantizer_cp_set(long jarg1, Level1Quantizer jarg1_, long jarg2, ClusteringParameters jarg2_); public final static native long Level1Quantizer_cp_get(long jarg1, Level1Quantizer jarg1_); public final static native void Level1Quantizer_clustering_index_set(long jarg1, Level1Quantizer jarg1_, long jarg2, Index jarg2_); public final static native long Level1Quantizer_clustering_index_get(long jarg1, Level1Quantizer jarg1_); public final static native void Level1Quantizer_train_q1(long jarg1, Level1Quantizer jarg1_, long jarg2, long jarg3, boolean jarg4, int jarg5); public final static native long Level1Quantizer_coarse_code_size(long jarg1, Level1Quantizer jarg1_); public final static native void Level1Quantizer_encode_listno(long jarg1, Level1Quantizer jarg1_, long jarg2, long jarg3); public final static native long Level1Quantizer_decode_listno(long jarg1, Level1Quantizer jarg1_, long jarg2); public final static native long new_Level1Quantizer__SWIG_0(long jarg1, Index jarg1_, long jarg2); public final static native long new_Level1Quantizer__SWIG_1(); public final static native void delete_Level1Quantizer(long jarg1); public final static native void IVFSearchParameters_nprobe_set(long jarg1, IVFSearchParameters jarg1_, long jarg2); public final static native long IVFSearchParameters_nprobe_get(long jarg1, IVFSearchParameters jarg1_); public final static native void IVFSearchParameters_max_codes_set(long jarg1, IVFSearchParameters jarg1_, long jarg2); public final static native long IVFSearchParameters_max_codes_get(long jarg1, IVFSearchParameters jarg1_); public final static native long new_IVFSearchParameters(); public final static native void delete_IVFSearchParameters(long jarg1); public final static native void IndexIVF_invlists_set(long jarg1, IndexIVF jarg1_, long jarg2, InvertedLists jarg2_); public final static native long IndexIVF_invlists_get(long jarg1, IndexIVF jarg1_); public final static native void IndexIVF_own_invlists_set(long jarg1, IndexIVF jarg1_, boolean jarg2); public final static native boolean IndexIVF_own_invlists_get(long jarg1, IndexIVF jarg1_); public final static native void IndexIVF_code_size_set(long jarg1, IndexIVF jarg1_, long jarg2); public final static native long IndexIVF_code_size_get(long jarg1, IndexIVF jarg1_); public final static native void IndexIVF_nprobe_set(long jarg1, IndexIVF jarg1_, long jarg2); public final static native long IndexIVF_nprobe_get(long jarg1, IndexIVF jarg1_); public final static native void IndexIVF_max_codes_set(long jarg1, IndexIVF jarg1_, long jarg2); public final static native long IndexIVF_max_codes_get(long jarg1, IndexIVF jarg1_); public final static native void IndexIVF_parallel_mode_set(long jarg1, IndexIVF jarg1_, int jarg2); public final static native int IndexIVF_parallel_mode_get(long jarg1, IndexIVF jarg1_); public final static native int IndexIVF_PARALLEL_MODE_NO_HEAP_INIT_get(long jarg1, IndexIVF jarg1_); public final static native void IndexIVF_direct_map_set(long jarg1, IndexIVF jarg1_, long jarg2); public final static native long IndexIVF_direct_map_get(long jarg1, IndexIVF jarg1_); public final static native void IndexIVF_reset(long jarg1, IndexIVF jarg1_); public final static native void IndexIVF_train(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3); public final static native void IndexIVF_add(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3); public final static native void IndexIVF_add_with_ids(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_); public final static native void IndexIVF_add_core(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, LongVector jarg5_); public final static native void IndexIVF_encode_vectors__SWIG_0(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, boolean jarg6); public final static native void IndexIVF_encode_vectors__SWIG_1(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5); public final static native void IndexIVF_add_sa_codes(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_); public final static native void IndexIVF_train_residual(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3); public final static native void IndexIVF_search_preassigned__SWIG_0(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, LongVector jarg5_, long jarg6, long jarg7, long jarg8, LongVector jarg8_, boolean jarg9, long jarg10, IVFSearchParameters jarg10_, long jarg11, IndexIVFStats jarg11_); public final static native void IndexIVF_search_preassigned__SWIG_1(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, LongVector jarg5_, long jarg6, long jarg7, long jarg8, LongVector jarg8_, boolean jarg9, long jarg10, IVFSearchParameters jarg10_); public final static native void IndexIVF_search_preassigned__SWIG_2(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, LongVector jarg5_, long jarg6, long jarg7, long jarg8, LongVector jarg8_, boolean jarg9); public final static native void IndexIVF_search(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexIVF_range_search(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, float jarg4, long jarg5, RangeSearchResult jarg5_); public final static native void IndexIVF_range_search_preassigned__SWIG_0(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, float jarg4, long jarg5, LongVector jarg5_, long jarg6, long jarg7, RangeSearchResult jarg7_, boolean jarg8, long jarg9, IVFSearchParameters jarg9_, long jarg10, IndexIVFStats jarg10_); public final static native void IndexIVF_range_search_preassigned__SWIG_1(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, float jarg4, long jarg5, LongVector jarg5_, long jarg6, long jarg7, RangeSearchResult jarg7_, boolean jarg8, long jarg9, IVFSearchParameters jarg9_); public final static native void IndexIVF_range_search_preassigned__SWIG_2(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, float jarg4, long jarg5, LongVector jarg5_, long jarg6, long jarg7, RangeSearchResult jarg7_, boolean jarg8); public final static native void IndexIVF_range_search_preassigned__SWIG_3(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, float jarg4, long jarg5, LongVector jarg5_, long jarg6, long jarg7, RangeSearchResult jarg7_); public final static native long IndexIVF_get_InvertedListScanner__SWIG_0(long jarg1, IndexIVF jarg1_, boolean jarg2); public final static native long IndexIVF_get_InvertedListScanner__SWIG_1(long jarg1, IndexIVF jarg1_); public final static native void IndexIVF_reconstruct(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3); public final static native void IndexIVF_update_vectors(long jarg1, IndexIVF jarg1_, int jarg2, long jarg3, LongVector jarg3_, long jarg4); public final static native void IndexIVF_reconstruct_n(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, long jarg4); public final static native void IndexIVF_search_and_reconstruct(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_, long jarg7); public final static native void IndexIVF_reconstruct_from_offset(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, long jarg4); public final static native long IndexIVF_remove_ids(long jarg1, IndexIVF jarg1_, long jarg2, IDSelector jarg2_); public final static native void IndexIVF_check_compatible_for_merge(long jarg1, IndexIVF jarg1_, long jarg2, IndexIVF jarg2_); public final static native void IndexIVF_merge_from(long jarg1, IndexIVF jarg1_, long jarg2, IndexIVF jarg2_, long jarg3); public final static native void IndexIVF_copy_subset_to(long jarg1, IndexIVF jarg1_, long jarg2, IndexIVF jarg2_, int jarg3, long jarg4, long jarg5); public final static native void delete_IndexIVF(long jarg1); public final static native long IndexIVF_get_list_size(long jarg1, IndexIVF jarg1_, long jarg2); public final static native void IndexIVF_make_direct_map__SWIG_0(long jarg1, IndexIVF jarg1_, boolean jarg2); public final static native void IndexIVF_make_direct_map__SWIG_1(long jarg1, IndexIVF jarg1_); public final static native void IndexIVF_set_direct_map_type(long jarg1, IndexIVF jarg1_, long jarg2); public final static native void IndexIVF_replace_invlists__SWIG_0(long jarg1, IndexIVF jarg1_, long jarg2, InvertedLists jarg2_, boolean jarg3); public final static native void IndexIVF_replace_invlists__SWIG_1(long jarg1, IndexIVF jarg1_, long jarg2, InvertedLists jarg2_); public final static native long IndexIVF_sa_code_size(long jarg1, IndexIVF jarg1_); public final static native void IndexIVF_sa_encode(long jarg1, IndexIVF jarg1_, long jarg2, long jarg3, long jarg4); public final static native void IndexIVFStats_nq_set(long jarg1, IndexIVFStats jarg1_, long jarg2); public final static native long IndexIVFStats_nq_get(long jarg1, IndexIVFStats jarg1_); public final static native void IndexIVFStats_nlist_set(long jarg1, IndexIVFStats jarg1_, long jarg2); public final static native long IndexIVFStats_nlist_get(long jarg1, IndexIVFStats jarg1_); public final static native void IndexIVFStats_ndis_set(long jarg1, IndexIVFStats jarg1_, long jarg2); public final static native long IndexIVFStats_ndis_get(long jarg1, IndexIVFStats jarg1_); public final static native void IndexIVFStats_nheap_updates_set(long jarg1, IndexIVFStats jarg1_, long jarg2); public final static native long IndexIVFStats_nheap_updates_get(long jarg1, IndexIVFStats jarg1_); public final static native void IndexIVFStats_quantization_time_set(long jarg1, IndexIVFStats jarg1_, double jarg2); public final static native double IndexIVFStats_quantization_time_get(long jarg1, IndexIVFStats jarg1_); public final static native void IndexIVFStats_search_time_set(long jarg1, IndexIVFStats jarg1_, double jarg2); public final static native double IndexIVFStats_search_time_get(long jarg1, IndexIVFStats jarg1_); public final static native long new_IndexIVFStats(); public final static native void IndexIVFStats_reset(long jarg1, IndexIVFStats jarg1_); public final static native void IndexIVFStats_add(long jarg1, IndexIVFStats jarg1_, long jarg2, IndexIVFStats jarg2_); public final static native void delete_IndexIVFStats(long jarg1); public final static native void indexIVF_stats_set(long jarg1, IndexIVFStats jarg1_); public final static native long indexIVF_stats_get(); public final static native short[] hamdis_tab_ham_bytes_get(); public final static native void HammingComputer4_a0_set(long jarg1, HammingComputer4 jarg1_, long jarg2); public final static native long HammingComputer4_a0_get(long jarg1, HammingComputer4 jarg1_); public final static native long new_HammingComputer4__SWIG_0(); public final static native long new_HammingComputer4__SWIG_1(long jarg1, int jarg2); public final static native void HammingComputer4_set(long jarg1, HammingComputer4 jarg1_, long jarg2, int jarg3); public final static native int HammingComputer4_hamming(long jarg1, HammingComputer4 jarg1_, long jarg2); public final static native void delete_HammingComputer4(long jarg1); public final static native void HammingComputer8_a0_set(long jarg1, HammingComputer8 jarg1_, long jarg2); public final static native long HammingComputer8_a0_get(long jarg1, HammingComputer8 jarg1_); public final static native long new_HammingComputer8__SWIG_0(); public final static native long new_HammingComputer8__SWIG_1(long jarg1, int jarg2); public final static native void HammingComputer8_set(long jarg1, HammingComputer8 jarg1_, long jarg2, int jarg3); public final static native int HammingComputer8_hamming(long jarg1, HammingComputer8 jarg1_, long jarg2); public final static native void delete_HammingComputer8(long jarg1); public final static native void HammingComputer16_a0_set(long jarg1, HammingComputer16 jarg1_, long jarg2); public final static native long HammingComputer16_a0_get(long jarg1, HammingComputer16 jarg1_); public final static native void HammingComputer16_a1_set(long jarg1, HammingComputer16 jarg1_, long jarg2); public final static native long HammingComputer16_a1_get(long jarg1, HammingComputer16 jarg1_); public final static native long new_HammingComputer16__SWIG_0(); public final static native long new_HammingComputer16__SWIG_1(long jarg1, int jarg2); public final static native void HammingComputer16_set(long jarg1, HammingComputer16 jarg1_, long jarg2, int jarg3); public final static native int HammingComputer16_hamming(long jarg1, HammingComputer16 jarg1_, long jarg2); public final static native void delete_HammingComputer16(long jarg1); public final static native void HammingComputer20_a0_set(long jarg1, HammingComputer20 jarg1_, long jarg2); public final static native long HammingComputer20_a0_get(long jarg1, HammingComputer20 jarg1_); public final static native void HammingComputer20_a1_set(long jarg1, HammingComputer20 jarg1_, long jarg2); public final static native long HammingComputer20_a1_get(long jarg1, HammingComputer20 jarg1_); public final static native void HammingComputer20_a2_set(long jarg1, HammingComputer20 jarg1_, long jarg2); public final static native long HammingComputer20_a2_get(long jarg1, HammingComputer20 jarg1_); public final static native long new_HammingComputer20__SWIG_0(); public final static native long new_HammingComputer20__SWIG_1(long jarg1, int jarg2); public final static native void HammingComputer20_set(long jarg1, HammingComputer20 jarg1_, long jarg2, int jarg3); public final static native int HammingComputer20_hamming(long jarg1, HammingComputer20 jarg1_, long jarg2); public final static native void delete_HammingComputer20(long jarg1); public final static native void HammingComputer32_a0_set(long jarg1, HammingComputer32 jarg1_, long jarg2); public final static native long HammingComputer32_a0_get(long jarg1, HammingComputer32 jarg1_); public final static native void HammingComputer32_a1_set(long jarg1, HammingComputer32 jarg1_, long jarg2); public final static native long HammingComputer32_a1_get(long jarg1, HammingComputer32 jarg1_); public final static native void HammingComputer32_a2_set(long jarg1, HammingComputer32 jarg1_, long jarg2); public final static native long HammingComputer32_a2_get(long jarg1, HammingComputer32 jarg1_); public final static native void HammingComputer32_a3_set(long jarg1, HammingComputer32 jarg1_, long jarg2); public final static native long HammingComputer32_a3_get(long jarg1, HammingComputer32 jarg1_); public final static native long new_HammingComputer32__SWIG_0(); public final static native long new_HammingComputer32__SWIG_1(long jarg1, int jarg2); public final static native void HammingComputer32_set(long jarg1, HammingComputer32 jarg1_, long jarg2, int jarg3); public final static native int HammingComputer32_hamming(long jarg1, HammingComputer32 jarg1_, long jarg2); public final static native void delete_HammingComputer32(long jarg1); public final static native void HammingComputer64_a0_set(long jarg1, HammingComputer64 jarg1_, long jarg2); public final static native long HammingComputer64_a0_get(long jarg1, HammingComputer64 jarg1_); public final static native void HammingComputer64_a1_set(long jarg1, HammingComputer64 jarg1_, long jarg2); public final static native long HammingComputer64_a1_get(long jarg1, HammingComputer64 jarg1_); public final static native void HammingComputer64_a2_set(long jarg1, HammingComputer64 jarg1_, long jarg2); public final static native long HammingComputer64_a2_get(long jarg1, HammingComputer64 jarg1_); public final static native void HammingComputer64_a3_set(long jarg1, HammingComputer64 jarg1_, long jarg2); public final static native long HammingComputer64_a3_get(long jarg1, HammingComputer64 jarg1_); public final static native void HammingComputer64_a4_set(long jarg1, HammingComputer64 jarg1_, long jarg2); public final static native long HammingComputer64_a4_get(long jarg1, HammingComputer64 jarg1_); public final static native void HammingComputer64_a5_set(long jarg1, HammingComputer64 jarg1_, long jarg2); public final static native long HammingComputer64_a5_get(long jarg1, HammingComputer64 jarg1_); public final static native void HammingComputer64_a6_set(long jarg1, HammingComputer64 jarg1_, long jarg2); public final static native long HammingComputer64_a6_get(long jarg1, HammingComputer64 jarg1_); public final static native void HammingComputer64_a7_set(long jarg1, HammingComputer64 jarg1_, long jarg2); public final static native long HammingComputer64_a7_get(long jarg1, HammingComputer64 jarg1_); public final static native long new_HammingComputer64__SWIG_0(); public final static native long new_HammingComputer64__SWIG_1(long jarg1, int jarg2); public final static native void HammingComputer64_set(long jarg1, HammingComputer64 jarg1_, long jarg2, int jarg3); public final static native int HammingComputer64_hamming(long jarg1, HammingComputer64 jarg1_, long jarg2); public final static native void delete_HammingComputer64(long jarg1); public final static native void HammingComputerDefault_a8_set(long jarg1, HammingComputerDefault jarg1_, long jarg2); public final static native long HammingComputerDefault_a8_get(long jarg1, HammingComputerDefault jarg1_); public final static native void HammingComputerDefault_quotient8_set(long jarg1, HammingComputerDefault jarg1_, int jarg2); public final static native int HammingComputerDefault_quotient8_get(long jarg1, HammingComputerDefault jarg1_); public final static native void HammingComputerDefault_remainder8_set(long jarg1, HammingComputerDefault jarg1_, int jarg2); public final static native int HammingComputerDefault_remainder8_get(long jarg1, HammingComputerDefault jarg1_); public final static native long new_HammingComputerDefault__SWIG_0(); public final static native long new_HammingComputerDefault__SWIG_1(long jarg1, int jarg2); public final static native void HammingComputerDefault_set(long jarg1, HammingComputerDefault jarg1_, long jarg2, int jarg3); public final static native int HammingComputerDefault_hamming(long jarg1, HammingComputerDefault jarg1_, long jarg2); public final static native void delete_HammingComputerDefault(long jarg1); public final static native void HammingComputerM8_a_set(long jarg1, HammingComputerM8 jarg1_, long jarg2); public final static native long HammingComputerM8_a_get(long jarg1, HammingComputerM8 jarg1_); public final static native void HammingComputerM8_n_set(long jarg1, HammingComputerM8 jarg1_, int jarg2); public final static native int HammingComputerM8_n_get(long jarg1, HammingComputerM8 jarg1_); public final static native long new_HammingComputerM8__SWIG_0(); public final static native long new_HammingComputerM8__SWIG_1(long jarg1, int jarg2); public final static native void HammingComputerM8_set(long jarg1, HammingComputerM8 jarg1_, long jarg2, int jarg3); public final static native int HammingComputerM8_hamming(long jarg1, HammingComputerM8 jarg1_, long jarg2); public final static native void delete_HammingComputerM8(long jarg1); public final static native void HammingComputerM4_a_set(long jarg1, HammingComputerM4 jarg1_, long jarg2); public final static native long HammingComputerM4_a_get(long jarg1, HammingComputerM4 jarg1_); public final static native void HammingComputerM4_n_set(long jarg1, HammingComputerM4 jarg1_, int jarg2); public final static native int HammingComputerM4_n_get(long jarg1, HammingComputerM4 jarg1_); public final static native long new_HammingComputerM4__SWIG_0(); public final static native long new_HammingComputerM4__SWIG_1(long jarg1, int jarg2); public final static native void HammingComputerM4_set(long jarg1, HammingComputerM4 jarg1_, long jarg2, int jarg3); public final static native int HammingComputerM4_hamming(long jarg1, HammingComputerM4 jarg1_, long jarg2); public final static native void delete_HammingComputerM4(long jarg1); public final static native int generalized_hamming_64(long jarg1); public final static native void GenHammingComputer8_a0_set(long jarg1, GenHammingComputer8 jarg1_, long jarg2); public final static native long GenHammingComputer8_a0_get(long jarg1, GenHammingComputer8 jarg1_); public final static native long new_GenHammingComputer8(long jarg1, int jarg2); public final static native int GenHammingComputer8_hamming(long jarg1, GenHammingComputer8 jarg1_, long jarg2); public final static native void delete_GenHammingComputer8(long jarg1); public final static native void GenHammingComputer16_a0_set(long jarg1, GenHammingComputer16 jarg1_, long jarg2); public final static native long GenHammingComputer16_a0_get(long jarg1, GenHammingComputer16 jarg1_); public final static native void GenHammingComputer16_a1_set(long jarg1, GenHammingComputer16 jarg1_, long jarg2); public final static native long GenHammingComputer16_a1_get(long jarg1, GenHammingComputer16 jarg1_); public final static native long new_GenHammingComputer16(long jarg1, int jarg2); public final static native int GenHammingComputer16_hamming(long jarg1, GenHammingComputer16 jarg1_, long jarg2); public final static native void delete_GenHammingComputer16(long jarg1); public final static native void GenHammingComputer32_a0_set(long jarg1, GenHammingComputer32 jarg1_, long jarg2); public final static native long GenHammingComputer32_a0_get(long jarg1, GenHammingComputer32 jarg1_); public final static native void GenHammingComputer32_a1_set(long jarg1, GenHammingComputer32 jarg1_, long jarg2); public final static native long GenHammingComputer32_a1_get(long jarg1, GenHammingComputer32 jarg1_); public final static native void GenHammingComputer32_a2_set(long jarg1, GenHammingComputer32 jarg1_, long jarg2); public final static native long GenHammingComputer32_a2_get(long jarg1, GenHammingComputer32 jarg1_); public final static native void GenHammingComputer32_a3_set(long jarg1, GenHammingComputer32 jarg1_, long jarg2); public final static native long GenHammingComputer32_a3_get(long jarg1, GenHammingComputer32 jarg1_); public final static native long new_GenHammingComputer32(long jarg1, int jarg2); public final static native int GenHammingComputer32_hamming(long jarg1, GenHammingComputer32 jarg1_, long jarg2); public final static native void delete_GenHammingComputer32(long jarg1); public final static native void GenHammingComputerM8_a_set(long jarg1, GenHammingComputerM8 jarg1_, long jarg2); public final static native long GenHammingComputerM8_a_get(long jarg1, GenHammingComputerM8 jarg1_); public final static native void GenHammingComputerM8_n_set(long jarg1, GenHammingComputerM8 jarg1_, int jarg2); public final static native int GenHammingComputerM8_n_get(long jarg1, GenHammingComputerM8 jarg1_); public final static native long new_GenHammingComputerM8(long jarg1, int jarg2); public final static native int GenHammingComputerM8_hamming(long jarg1, GenHammingComputerM8 jarg1_, long jarg2); public final static native void delete_GenHammingComputerM8(long jarg1); public final static native void generalized_hammings_knn_hc__SWIG_0(long jarg1, long jarg2, long jarg3, long jarg4, long jarg5, int jarg6); public final static native void generalized_hammings_knn_hc__SWIG_1(long jarg1, long jarg2, long jarg3, long jarg4, long jarg5); public final static native void check_compatible_for_merge(long jarg1, Index jarg1_, long jarg2, Index jarg2_); public final static native long extract_index_ivf__SWIG_0(long jarg1, Index jarg1_); public final static native long try_extract_index_ivf__SWIG_0(long jarg1, Index jarg1_); public final static native void merge_into(long jarg1, Index jarg1_, long jarg2, Index jarg2_, boolean jarg3); public final static native void search_centroid(long jarg1, Index jarg1_, long jarg2, int jarg3, long jarg4, LongVector jarg4_); public final static native void search_and_return_centroids(long jarg1, Index jarg1_, long jarg2, long jarg3, int jarg4, long jarg5, long jarg6, LongVector jarg6_, long jarg7, LongVector jarg7_, long jarg8, LongVector jarg8_); public final static native void SlidingIndexWindow_index_set(long jarg1, SlidingIndexWindow jarg1_, long jarg2, Index jarg2_); public final static native long SlidingIndexWindow_index_get(long jarg1, SlidingIndexWindow jarg1_); public final static native void SlidingIndexWindow_ils_set(long jarg1, SlidingIndexWindow jarg1_, long jarg2, ArrayInvertedLists jarg2_); public final static native long SlidingIndexWindow_ils_get(long jarg1, SlidingIndexWindow jarg1_); public final static native void SlidingIndexWindow_n_slice_set(long jarg1, SlidingIndexWindow jarg1_, int jarg2); public final static native int SlidingIndexWindow_n_slice_get(long jarg1, SlidingIndexWindow jarg1_); public final static native void SlidingIndexWindow_nlist_set(long jarg1, SlidingIndexWindow jarg1_, long jarg2); public final static native long SlidingIndexWindow_nlist_get(long jarg1, SlidingIndexWindow jarg1_); public final static native void SlidingIndexWindow_sizes_set(long jarg1, SlidingIndexWindow jarg1_, long jarg2); public final static native long SlidingIndexWindow_sizes_get(long jarg1, SlidingIndexWindow jarg1_); public final static native long new_SlidingIndexWindow(long jarg1, Index jarg1_); public final static native void SlidingIndexWindow_step(long jarg1, SlidingIndexWindow jarg1_, long jarg2, Index jarg2_, boolean jarg3); public final static native void delete_SlidingIndexWindow(long jarg1); public final static native long get_invlist_range(long jarg1, Index jarg1_, int jarg2, int jarg3); public final static native void set_invlist_range(long jarg1, Index jarg1_, int jarg2, int jarg3, long jarg4, ArrayInvertedLists jarg4_); public final static native void search_with_parameters__SWIG_0(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_, long jarg7, IVFSearchParameters jarg7_, long jarg8, long jarg9); public final static native void search_with_parameters__SWIG_1(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_, long jarg7, IVFSearchParameters jarg7_, long jarg8); public final static native void search_with_parameters__SWIG_2(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_, long jarg7, IVFSearchParameters jarg7_); public final static native void range_search_with_parameters__SWIG_0(long jarg1, Index jarg1_, long jarg2, long jarg3, float jarg4, long jarg5, RangeSearchResult jarg5_, long jarg6, IVFSearchParameters jarg6_, long jarg7, long jarg8); public final static native void range_search_with_parameters__SWIG_1(long jarg1, Index jarg1_, long jarg2, long jarg3, float jarg4, long jarg5, RangeSearchResult jarg5_, long jarg6, IVFSearchParameters jarg6_, long jarg7); public final static native void range_search_with_parameters__SWIG_2(long jarg1, Index jarg1_, long jarg2, long jarg3, float jarg4, long jarg5, RangeSearchResult jarg5_, long jarg6, IVFSearchParameters jarg6_); public final static native void IndexScalarQuantizer_sq_set(long jarg1, IndexScalarQuantizer jarg1_, long jarg2); public final static native long IndexScalarQuantizer_sq_get(long jarg1, IndexScalarQuantizer jarg1_); public final static native long new_IndexScalarQuantizer__SWIG_0(int jarg1, long jarg2, int jarg3); public final static native long new_IndexScalarQuantizer__SWIG_1(int jarg1, long jarg2); public final static native long new_IndexScalarQuantizer__SWIG_2(); public final static native void IndexScalarQuantizer_train(long jarg1, IndexScalarQuantizer jarg1_, long jarg2, long jarg3); public final static native void IndexScalarQuantizer_search(long jarg1, IndexScalarQuantizer jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native long IndexScalarQuantizer_get_distance_computer(long jarg1, IndexScalarQuantizer jarg1_); public final static native void IndexScalarQuantizer_sa_encode(long jarg1, IndexScalarQuantizer jarg1_, long jarg2, long jarg3, long jarg4); public final static native void IndexScalarQuantizer_sa_decode(long jarg1, IndexScalarQuantizer jarg1_, long jarg2, long jarg3, long jarg4); public final static native void delete_IndexScalarQuantizer(long jarg1); public final static native void IndexIVFScalarQuantizer_sq_set(long jarg1, IndexIVFScalarQuantizer jarg1_, long jarg2); public final static native long IndexIVFScalarQuantizer_sq_get(long jarg1, IndexIVFScalarQuantizer jarg1_); public final static native void IndexIVFScalarQuantizer_by_residual_set(long jarg1, IndexIVFScalarQuantizer jarg1_, boolean jarg2); public final static native boolean IndexIVFScalarQuantizer_by_residual_get(long jarg1, IndexIVFScalarQuantizer jarg1_); public final static native long new_IndexIVFScalarQuantizer__SWIG_0(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4, int jarg5, boolean jarg6); public final static native long new_IndexIVFScalarQuantizer__SWIG_1(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4, int jarg5); public final static native long new_IndexIVFScalarQuantizer__SWIG_2(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4); public final static native long new_IndexIVFScalarQuantizer__SWIG_3(); public final static native void IndexIVFScalarQuantizer_train_residual(long jarg1, IndexIVFScalarQuantizer jarg1_, long jarg2, long jarg3); public final static native void IndexIVFScalarQuantizer_encode_vectors__SWIG_0(long jarg1, IndexIVFScalarQuantizer jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, boolean jarg6); public final static native void IndexIVFScalarQuantizer_encode_vectors__SWIG_1(long jarg1, IndexIVFScalarQuantizer jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5); public final static native void IndexIVFScalarQuantizer_add_core(long jarg1, IndexIVFScalarQuantizer jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, LongVector jarg5_); public final static native long IndexIVFScalarQuantizer_get_InvertedListScanner(long jarg1, IndexIVFScalarQuantizer jarg1_, boolean jarg2); public final static native void IndexIVFScalarQuantizer_reconstruct_from_offset(long jarg1, IndexIVFScalarQuantizer jarg1_, long jarg2, long jarg3, long jarg4); public final static native void IndexIVFScalarQuantizer_sa_decode(long jarg1, IndexIVFScalarQuantizer jarg1_, long jarg2, long jarg3, long jarg4); public final static native void delete_IndexIVFScalarQuantizer(long jarg1); public final static native void HNSW_MinimaxHeap_n_set(long jarg1, HNSW.MinimaxHeap jarg1_, int jarg2); public final static native int HNSW_MinimaxHeap_n_get(long jarg1, HNSW.MinimaxHeap jarg1_); public final static native void HNSW_MinimaxHeap_k_set(long jarg1, HNSW.MinimaxHeap jarg1_, int jarg2); public final static native int HNSW_MinimaxHeap_k_get(long jarg1, HNSW.MinimaxHeap jarg1_); public final static native void HNSW_MinimaxHeap_nvalid_set(long jarg1, HNSW.MinimaxHeap jarg1_, int jarg2); public final static native int HNSW_MinimaxHeap_nvalid_get(long jarg1, HNSW.MinimaxHeap jarg1_); public final static native void HNSW_MinimaxHeap_ids_set(long jarg1, HNSW.MinimaxHeap jarg1_, long jarg2, IntVector jarg2_); public final static native long HNSW_MinimaxHeap_ids_get(long jarg1, HNSW.MinimaxHeap jarg1_); public final static native void HNSW_MinimaxHeap_dis_set(long jarg1, HNSW.MinimaxHeap jarg1_, long jarg2, FloatVector jarg2_); public final static native long HNSW_MinimaxHeap_dis_get(long jarg1, HNSW.MinimaxHeap jarg1_); public final static native long new_HNSW_MinimaxHeap(int jarg1); public final static native void HNSW_MinimaxHeap_push(long jarg1, HNSW.MinimaxHeap jarg1_, int jarg2, float jarg3); public final static native float HNSW_MinimaxHeap_max(long jarg1, HNSW.MinimaxHeap jarg1_); public final static native int HNSW_MinimaxHeap_size(long jarg1, HNSW.MinimaxHeap jarg1_); public final static native void HNSW_MinimaxHeap_clear(long jarg1, HNSW.MinimaxHeap jarg1_); public final static native int HNSW_MinimaxHeap_pop_min__SWIG_0(long jarg1, HNSW.MinimaxHeap jarg1_, long jarg2); public final static native int HNSW_MinimaxHeap_pop_min__SWIG_1(long jarg1, HNSW.MinimaxHeap jarg1_); public final static native int HNSW_MinimaxHeap_count_below(long jarg1, HNSW.MinimaxHeap jarg1_, float jarg2); public final static native void delete_HNSW_MinimaxHeap(long jarg1); public final static native void HNSW_NodeDistCloser_d_set(long jarg1, HNSW.NodeDistCloser jarg1_, float jarg2); public final static native float HNSW_NodeDistCloser_d_get(long jarg1, HNSW.NodeDistCloser jarg1_); public final static native void HNSW_NodeDistCloser_id_set(long jarg1, HNSW.NodeDistCloser jarg1_, int jarg2); public final static native int HNSW_NodeDistCloser_id_get(long jarg1, HNSW.NodeDistCloser jarg1_); public final static native long new_HNSW_NodeDistCloser(float jarg1, int jarg2); public final static native void delete_HNSW_NodeDistCloser(long jarg1); public final static native void HNSW_NodeDistFarther_d_set(long jarg1, HNSW.NodeDistFarther jarg1_, float jarg2); public final static native float HNSW_NodeDistFarther_d_get(long jarg1, HNSW.NodeDistFarther jarg1_); public final static native void HNSW_NodeDistFarther_id_set(long jarg1, HNSW.NodeDistFarther jarg1_, int jarg2); public final static native int HNSW_NodeDistFarther_id_get(long jarg1, HNSW.NodeDistFarther jarg1_); public final static native long new_HNSW_NodeDistFarther(float jarg1, int jarg2); public final static native void delete_HNSW_NodeDistFarther(long jarg1); public final static native void HNSW_assign_probas_set(long jarg1, HNSW jarg1_, long jarg2, DoubleVector jarg2_); public final static native long HNSW_assign_probas_get(long jarg1, HNSW jarg1_); public final static native void HNSW_cum_nneighbor_per_level_set(long jarg1, HNSW jarg1_, long jarg2, IntVector jarg2_); public final static native long HNSW_cum_nneighbor_per_level_get(long jarg1, HNSW jarg1_); public final static native void HNSW_levels_set(long jarg1, HNSW jarg1_, long jarg2, IntVector jarg2_); public final static native long HNSW_levels_get(long jarg1, HNSW jarg1_); public final static native void HNSW_offsets_set(long jarg1, HNSW jarg1_, long jarg2, Uint64Vector jarg2_); public final static native long HNSW_offsets_get(long jarg1, HNSW jarg1_); public final static native void HNSW_neighbors_set(long jarg1, HNSW jarg1_, long jarg2, IntVector jarg2_); public final static native long HNSW_neighbors_get(long jarg1, HNSW jarg1_); public final static native void HNSW_entry_point_set(long jarg1, HNSW jarg1_, int jarg2); public final static native int HNSW_entry_point_get(long jarg1, HNSW jarg1_); public final static native void HNSW_rng_set(long jarg1, HNSW jarg1_, long jarg2); public final static native long HNSW_rng_get(long jarg1, HNSW jarg1_); public final static native void HNSW_max_level_set(long jarg1, HNSW jarg1_, int jarg2); public final static native int HNSW_max_level_get(long jarg1, HNSW jarg1_); public final static native void HNSW_efConstruction_set(long jarg1, HNSW jarg1_, int jarg2); public final static native int HNSW_efConstruction_get(long jarg1, HNSW jarg1_); public final static native void HNSW_efSearch_set(long jarg1, HNSW jarg1_, int jarg2); public final static native int HNSW_efSearch_get(long jarg1, HNSW jarg1_); public final static native void HNSW_check_relative_distance_set(long jarg1, HNSW jarg1_, boolean jarg2); public final static native boolean HNSW_check_relative_distance_get(long jarg1, HNSW jarg1_); public final static native void HNSW_upper_beam_set(long jarg1, HNSW jarg1_, int jarg2); public final static native int HNSW_upper_beam_get(long jarg1, HNSW jarg1_); public final static native void HNSW_search_bounded_queue_set(long jarg1, HNSW jarg1_, boolean jarg2); public final static native boolean HNSW_search_bounded_queue_get(long jarg1, HNSW jarg1_); public final static native void HNSW_set_default_probas(long jarg1, HNSW jarg1_, int jarg2, float jarg3); public final static native void HNSW_set_nb_neighbors(long jarg1, HNSW jarg1_, int jarg2, int jarg3); public final static native int HNSW_nb_neighbors(long jarg1, HNSW jarg1_, int jarg2); public final static native int HNSW_cum_nb_neighbors(long jarg1, HNSW jarg1_, int jarg2); public final static native void HNSW_neighbor_range(long jarg1, HNSW jarg1_, long jarg2, int jarg3, long jarg4, long jarg5); public final static native long new_HNSW__SWIG_0(int jarg1); public final static native long new_HNSW__SWIG_1(); public final static native int HNSW_random_level(long jarg1, HNSW jarg1_); public final static native void HNSW_fill_with_random_links(long jarg1, HNSW jarg1_, long jarg2); public final static native void HNSW_add_links_starting_from(long jarg1, HNSW jarg1_, long jarg2, DistanceComputer jarg2_, int jarg3, int jarg4, float jarg5, int jarg6, long jarg7, long jarg8, VisitedTable jarg8_); public final static native void HNSW_add_with_locks(long jarg1, HNSW jarg1_, long jarg2, DistanceComputer jarg2_, int jarg3, int jarg4, long jarg5, long jarg6, VisitedTable jarg6_); public final static native int HNSW_search_from_candidates__SWIG_0(long jarg1, HNSW jarg1_, long jarg2, DistanceComputer jarg2_, int jarg3, long jarg4, LongVector jarg4_, long jarg5, long jarg6, HNSW.MinimaxHeap jarg6_, long jarg7, VisitedTable jarg7_, long jarg8, HNSWStats jarg8_, int jarg9, int jarg10); public final static native int HNSW_search_from_candidates__SWIG_1(long jarg1, HNSW jarg1_, long jarg2, DistanceComputer jarg2_, int jarg3, long jarg4, LongVector jarg4_, long jarg5, long jarg6, HNSW.MinimaxHeap jarg6_, long jarg7, VisitedTable jarg7_, long jarg8, HNSWStats jarg8_, int jarg9); public final static native long HNSW_search_from_candidate_unbounded(long jarg1, HNSW jarg1_, long jarg2, long jarg3, DistanceComputer jarg3_, int jarg4, long jarg5, VisitedTable jarg5_, long jarg6, HNSWStats jarg6_); public final static native long HNSW_search(long jarg1, HNSW jarg1_, long jarg2, DistanceComputer jarg2_, int jarg3, long jarg4, LongVector jarg4_, long jarg5, long jarg6, VisitedTable jarg6_); public final static native void HNSW_reset(long jarg1, HNSW jarg1_); public final static native void HNSW_clear_neighbor_tables(long jarg1, HNSW jarg1_, int jarg2); public final static native void HNSW_print_neighbor_stats(long jarg1, HNSW jarg1_, int jarg2); public final static native int HNSW_prepare_level_tab__SWIG_0(long jarg1, HNSW jarg1_, long jarg2, boolean jarg3); public final static native int HNSW_prepare_level_tab__SWIG_1(long jarg1, HNSW jarg1_, long jarg2); public final static native void HNSW_shrink_neighbor_list(long jarg1, DistanceComputer jarg1_, long jarg2, long jarg3, int jarg4); public final static native void delete_HNSW(long jarg1); public final static native void HNSWStats_n1_set(long jarg1, HNSWStats jarg1_, long jarg2); public final static native long HNSWStats_n1_get(long jarg1, HNSWStats jarg1_); public final static native void HNSWStats_n2_set(long jarg1, HNSWStats jarg1_, long jarg2); public final static native long HNSWStats_n2_get(long jarg1, HNSWStats jarg1_); public final static native void HNSWStats_n3_set(long jarg1, HNSWStats jarg1_, long jarg2); public final static native long HNSWStats_n3_get(long jarg1, HNSWStats jarg1_); public final static native void HNSWStats_ndis_set(long jarg1, HNSWStats jarg1_, long jarg2); public final static native long HNSWStats_ndis_get(long jarg1, HNSWStats jarg1_); public final static native void HNSWStats_nreorder_set(long jarg1, HNSWStats jarg1_, long jarg2); public final static native long HNSWStats_nreorder_get(long jarg1, HNSWStats jarg1_); public final static native long new_HNSWStats__SWIG_0(long jarg1, long jarg2, long jarg3, long jarg4, long jarg5); public final static native long new_HNSWStats__SWIG_1(long jarg1, long jarg2, long jarg3, long jarg4); public final static native long new_HNSWStats__SWIG_2(long jarg1, long jarg2, long jarg3); public final static native long new_HNSWStats__SWIG_3(long jarg1, long jarg2); public final static native long new_HNSWStats__SWIG_4(long jarg1); public final static native long new_HNSWStats__SWIG_5(); public final static native void HNSWStats_reset(long jarg1, HNSWStats jarg1_); public final static native void HNSWStats_combine(long jarg1, HNSWStats jarg1_, long jarg2, HNSWStats jarg2_); public final static native void delete_HNSWStats(long jarg1); public final static native void hnsw_stats_set(long jarg1, HNSWStats jarg1_); public final static native long hnsw_stats_get(); public final static native long ReconstructFromNeighbors_index_get(long jarg1, ReconstructFromNeighbors jarg1_); public final static native void ReconstructFromNeighbors_M_set(long jarg1, ReconstructFromNeighbors jarg1_, long jarg2); public final static native long ReconstructFromNeighbors_M_get(long jarg1, ReconstructFromNeighbors jarg1_); public final static native void ReconstructFromNeighbors_k_set(long jarg1, ReconstructFromNeighbors jarg1_, long jarg2); public final static native long ReconstructFromNeighbors_k_get(long jarg1, ReconstructFromNeighbors jarg1_); public final static native void ReconstructFromNeighbors_nsq_set(long jarg1, ReconstructFromNeighbors jarg1_, long jarg2); public final static native long ReconstructFromNeighbors_nsq_get(long jarg1, ReconstructFromNeighbors jarg1_); public final static native void ReconstructFromNeighbors_code_size_set(long jarg1, ReconstructFromNeighbors jarg1_, long jarg2); public final static native long ReconstructFromNeighbors_code_size_get(long jarg1, ReconstructFromNeighbors jarg1_); public final static native void ReconstructFromNeighbors_k_reorder_set(long jarg1, ReconstructFromNeighbors jarg1_, int jarg2); public final static native int ReconstructFromNeighbors_k_reorder_get(long jarg1, ReconstructFromNeighbors jarg1_); public final static native void ReconstructFromNeighbors_codebook_set(long jarg1, ReconstructFromNeighbors jarg1_, long jarg2, FloatVector jarg2_); public final static native long ReconstructFromNeighbors_codebook_get(long jarg1, ReconstructFromNeighbors jarg1_); public final static native void ReconstructFromNeighbors_codes_set(long jarg1, ReconstructFromNeighbors jarg1_, long jarg2, ByteVector jarg2_); public final static native long ReconstructFromNeighbors_codes_get(long jarg1, ReconstructFromNeighbors jarg1_); public final static native void ReconstructFromNeighbors_ntotal_set(long jarg1, ReconstructFromNeighbors jarg1_, long jarg2); public final static native long ReconstructFromNeighbors_ntotal_get(long jarg1, ReconstructFromNeighbors jarg1_); public final static native void ReconstructFromNeighbors_d_set(long jarg1, ReconstructFromNeighbors jarg1_, long jarg2); public final static native long ReconstructFromNeighbors_d_get(long jarg1, ReconstructFromNeighbors jarg1_); public final static native void ReconstructFromNeighbors_dsub_set(long jarg1, ReconstructFromNeighbors jarg1_, long jarg2); public final static native long ReconstructFromNeighbors_dsub_get(long jarg1, ReconstructFromNeighbors jarg1_); public final static native long new_ReconstructFromNeighbors__SWIG_0(long jarg1, IndexHNSW jarg1_, long jarg2, long jarg3); public final static native long new_ReconstructFromNeighbors__SWIG_1(long jarg1, IndexHNSW jarg1_, long jarg2); public final static native long new_ReconstructFromNeighbors__SWIG_2(long jarg1, IndexHNSW jarg1_); public final static native void ReconstructFromNeighbors_add_codes(long jarg1, ReconstructFromNeighbors jarg1_, long jarg2, long jarg3); public final static native long ReconstructFromNeighbors_compute_distances(long jarg1, ReconstructFromNeighbors jarg1_, long jarg2, long jarg3, LongVector jarg3_, long jarg4, long jarg5); public final static native void ReconstructFromNeighbors_estimate_code(long jarg1, ReconstructFromNeighbors jarg1_, long jarg2, int jarg3, long jarg4); public final static native void ReconstructFromNeighbors_reconstruct(long jarg1, ReconstructFromNeighbors jarg1_, int jarg2, long jarg3, long jarg4); public final static native void ReconstructFromNeighbors_reconstruct_n(long jarg1, ReconstructFromNeighbors jarg1_, int jarg2, int jarg3, long jarg4); public final static native void ReconstructFromNeighbors_get_neighbor_table(long jarg1, ReconstructFromNeighbors jarg1_, int jarg2, long jarg3); public final static native void delete_ReconstructFromNeighbors(long jarg1); public final static native void IndexHNSW_hnsw_set(long jarg1, IndexHNSW jarg1_, long jarg2, HNSW jarg2_); public final static native long IndexHNSW_hnsw_get(long jarg1, IndexHNSW jarg1_); public final static native void IndexHNSW_own_fields_set(long jarg1, IndexHNSW jarg1_, boolean jarg2); public final static native boolean IndexHNSW_own_fields_get(long jarg1, IndexHNSW jarg1_); public final static native void IndexHNSW_storage_set(long jarg1, IndexHNSW jarg1_, long jarg2, Index jarg2_); public final static native long IndexHNSW_storage_get(long jarg1, IndexHNSW jarg1_); public final static native void IndexHNSW_reconstruct_from_neighbors_set(long jarg1, IndexHNSW jarg1_, long jarg2, ReconstructFromNeighbors jarg2_); public final static native long IndexHNSW_reconstruct_from_neighbors_get(long jarg1, IndexHNSW jarg1_); public final static native long new_IndexHNSW__SWIG_0(int jarg1, int jarg2, int jarg3); public final static native long new_IndexHNSW__SWIG_1(int jarg1, int jarg2); public final static native long new_IndexHNSW__SWIG_2(int jarg1); public final static native long new_IndexHNSW__SWIG_3(); public final static native long new_IndexHNSW__SWIG_4(long jarg1, Index jarg1_, int jarg2); public final static native long new_IndexHNSW__SWIG_5(long jarg1, Index jarg1_); public final static native void delete_IndexHNSW(long jarg1); public final static native void IndexHNSW_add(long jarg1, IndexHNSW jarg1_, long jarg2, long jarg3); public final static native void IndexHNSW_train(long jarg1, IndexHNSW jarg1_, long jarg2, long jarg3); public final static native void IndexHNSW_search(long jarg1, IndexHNSW jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexHNSW_reconstruct(long jarg1, IndexHNSW jarg1_, long jarg2, long jarg3); public final static native void IndexHNSW_reset(long jarg1, IndexHNSW jarg1_); public final static native void IndexHNSW_shrink_level_0_neighbors(long jarg1, IndexHNSW jarg1_, int jarg2); public final static native void IndexHNSW_search_level_0__SWIG_0(long jarg1, IndexHNSW jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, long jarg7, long jarg8, LongVector jarg8_, int jarg9, int jarg10); public final static native void IndexHNSW_search_level_0__SWIG_1(long jarg1, IndexHNSW jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, long jarg7, long jarg8, LongVector jarg8_, int jarg9); public final static native void IndexHNSW_search_level_0__SWIG_2(long jarg1, IndexHNSW jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, long jarg7, long jarg8, LongVector jarg8_); public final static native void IndexHNSW_init_level_0_from_knngraph(long jarg1, IndexHNSW jarg1_, int jarg2, long jarg3, long jarg4, LongVector jarg4_); public final static native void IndexHNSW_init_level_0_from_entry_points(long jarg1, IndexHNSW jarg1_, int jarg2, long jarg3, long jarg4); public final static native void IndexHNSW_reorder_links(long jarg1, IndexHNSW jarg1_); public final static native void IndexHNSW_link_singletons(long jarg1, IndexHNSW jarg1_); public final static native long new_IndexHNSWFlat__SWIG_0(); public final static native long new_IndexHNSWFlat__SWIG_1(int jarg1, int jarg2, int jarg3); public final static native long new_IndexHNSWFlat__SWIG_2(int jarg1, int jarg2); public final static native void delete_IndexHNSWFlat(long jarg1); public final static native long new_IndexHNSWPQ__SWIG_0(); public final static native long new_IndexHNSWPQ__SWIG_1(int jarg1, int jarg2, int jarg3); public final static native void IndexHNSWPQ_train(long jarg1, IndexHNSWPQ jarg1_, long jarg2, long jarg3); public final static native void delete_IndexHNSWPQ(long jarg1); public final static native long new_IndexHNSWSQ__SWIG_0(); public final static native long new_IndexHNSWSQ__SWIG_1(int jarg1, long jarg2, int jarg3, int jarg4); public final static native long new_IndexHNSWSQ__SWIG_2(int jarg1, long jarg2, int jarg3); public final static native void delete_IndexHNSWSQ(long jarg1); public final static native long new_IndexHNSW2Level__SWIG_0(); public final static native long new_IndexHNSW2Level__SWIG_1(long jarg1, Index jarg1_, long jarg2, int jarg3, int jarg4); public final static native void IndexHNSW2Level_flip_to_ivf(long jarg1, IndexHNSW2Level jarg1_); public final static native void IndexHNSW2Level_search(long jarg1, IndexHNSW2Level jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void delete_IndexHNSW2Level(long jarg1); public final static native long new_IndexIVFFlat__SWIG_0(long jarg1, Index jarg1_, long jarg2, long jarg3, int jarg4); public final static native long new_IndexIVFFlat__SWIG_1(long jarg1, Index jarg1_, long jarg2, long jarg3); public final static native void IndexIVFFlat_add_core(long jarg1, IndexIVFFlat jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, LongVector jarg5_); public final static native void IndexIVFFlat_encode_vectors__SWIG_0(long jarg1, IndexIVFFlat jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, boolean jarg6); public final static native void IndexIVFFlat_encode_vectors__SWIG_1(long jarg1, IndexIVFFlat jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5); public final static native long IndexIVFFlat_get_InvertedListScanner(long jarg1, IndexIVFFlat jarg1_, boolean jarg2); public final static native void IndexIVFFlat_reconstruct_from_offset(long jarg1, IndexIVFFlat jarg1_, long jarg2, long jarg3, long jarg4); public final static native void IndexIVFFlat_sa_decode(long jarg1, IndexIVFFlat jarg1_, long jarg2, long jarg3, long jarg4); public final static native long new_IndexIVFFlat__SWIG_2(); public final static native void delete_IndexIVFFlat(long jarg1); public final static native void IndexIVFFlatDedup_instances_set(long jarg1, IndexIVFFlatDedup jarg1_, long jarg2); public final static native long IndexIVFFlatDedup_instances_get(long jarg1, IndexIVFFlatDedup jarg1_); public final static native long new_IndexIVFFlatDedup__SWIG_0(long jarg1, Index jarg1_, long jarg2, long jarg3, int jarg4); public final static native long new_IndexIVFFlatDedup__SWIG_1(long jarg1, Index jarg1_, long jarg2, long jarg3); public final static native void IndexIVFFlatDedup_train(long jarg1, IndexIVFFlatDedup jarg1_, long jarg2, long jarg3); public final static native void IndexIVFFlatDedup_add_with_ids(long jarg1, IndexIVFFlatDedup jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_); public final static native void IndexIVFFlatDedup_search_preassigned__SWIG_0(long jarg1, IndexIVFFlatDedup jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, LongVector jarg5_, long jarg6, long jarg7, long jarg8, LongVector jarg8_, boolean jarg9, long jarg10, IVFSearchParameters jarg10_, long jarg11, IndexIVFStats jarg11_); public final static native void IndexIVFFlatDedup_search_preassigned__SWIG_1(long jarg1, IndexIVFFlatDedup jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, LongVector jarg5_, long jarg6, long jarg7, long jarg8, LongVector jarg8_, boolean jarg9, long jarg10, IVFSearchParameters jarg10_); public final static native void IndexIVFFlatDedup_search_preassigned__SWIG_2(long jarg1, IndexIVFFlatDedup jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, LongVector jarg5_, long jarg6, long jarg7, long jarg8, LongVector jarg8_, boolean jarg9); public final static native long IndexIVFFlatDedup_remove_ids(long jarg1, IndexIVFFlatDedup jarg1_, long jarg2, IDSelector jarg2_); public final static native void IndexIVFFlatDedup_range_search(long jarg1, IndexIVFFlatDedup jarg1_, long jarg2, long jarg3, float jarg4, long jarg5, RangeSearchResult jarg5_); public final static native void IndexIVFFlatDedup_update_vectors(long jarg1, IndexIVFFlatDedup jarg1_, int jarg2, long jarg3, LongVector jarg3_, long jarg4); public final static native void IndexIVFFlatDedup_reconstruct_from_offset(long jarg1, IndexIVFFlatDedup jarg1_, long jarg2, long jarg3, long jarg4); public final static native long new_IndexIVFFlatDedup__SWIG_2(); public final static native void delete_IndexIVFFlatDedup(long jarg1); public final static native void OnDiskOneList_size_set(long jarg1, OnDiskOneList jarg1_, long jarg2); public final static native long OnDiskOneList_size_get(long jarg1, OnDiskOneList jarg1_); public final static native void OnDiskOneList_capacity_set(long jarg1, OnDiskOneList jarg1_, long jarg2); public final static native long OnDiskOneList_capacity_get(long jarg1, OnDiskOneList jarg1_); public final static native void OnDiskOneList_offset_set(long jarg1, OnDiskOneList jarg1_, long jarg2); public final static native long OnDiskOneList_offset_get(long jarg1, OnDiskOneList jarg1_); public final static native long new_OnDiskOneList(); public final static native void delete_OnDiskOneList(long jarg1); public final static native void OnDiskInvertedLists_lists_set(long jarg1, OnDiskInvertedLists jarg1_, long jarg2); public final static native long OnDiskInvertedLists_lists_get(long jarg1, OnDiskInvertedLists jarg1_); public final static native void OnDiskInvertedLists_Slot_offset_set(long jarg1, OnDiskInvertedLists.Slot jarg1_, long jarg2); public final static native long OnDiskInvertedLists_Slot_offset_get(long jarg1, OnDiskInvertedLists.Slot jarg1_); public final static native void OnDiskInvertedLists_Slot_capacity_set(long jarg1, OnDiskInvertedLists.Slot jarg1_, long jarg2); public final static native long OnDiskInvertedLists_Slot_capacity_get(long jarg1, OnDiskInvertedLists.Slot jarg1_); public final static native long new_OnDiskInvertedLists_Slot__SWIG_0(long jarg1, long jarg2); public final static native long new_OnDiskInvertedLists_Slot__SWIG_1(); public final static native void delete_OnDiskInvertedLists_Slot(long jarg1); public final static native void OnDiskInvertedLists_slots_set(long jarg1, OnDiskInvertedLists jarg1_, long jarg2); public final static native long OnDiskInvertedLists_slots_get(long jarg1, OnDiskInvertedLists jarg1_); public final static native void OnDiskInvertedLists_filename_set(long jarg1, OnDiskInvertedLists jarg1_, String jarg2); public final static native String OnDiskInvertedLists_filename_get(long jarg1, OnDiskInvertedLists jarg1_); public final static native void OnDiskInvertedLists_totsize_set(long jarg1, OnDiskInvertedLists jarg1_, long jarg2); public final static native long OnDiskInvertedLists_totsize_get(long jarg1, OnDiskInvertedLists jarg1_); public final static native void OnDiskInvertedLists_ptr_set(long jarg1, OnDiskInvertedLists jarg1_, long jarg2); public final static native long OnDiskInvertedLists_ptr_get(long jarg1, OnDiskInvertedLists jarg1_); public final static native void OnDiskInvertedLists_read_only_set(long jarg1, OnDiskInvertedLists jarg1_, boolean jarg2); public final static native boolean OnDiskInvertedLists_read_only_get(long jarg1, OnDiskInvertedLists jarg1_); public final static native long new_OnDiskInvertedLists__SWIG_0(long jarg1, long jarg2, String jarg3); public final static native long OnDiskInvertedLists_list_size(long jarg1, OnDiskInvertedLists jarg1_, long jarg2); public final static native long OnDiskInvertedLists_get_codes(long jarg1, OnDiskInvertedLists jarg1_, long jarg2); public final static native long OnDiskInvertedLists_get_ids(long jarg1, OnDiskInvertedLists jarg1_, long jarg2); public final static native long OnDiskInvertedLists_add_entries(long jarg1, OnDiskInvertedLists jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5); public final static native void OnDiskInvertedLists_update_entries(long jarg1, OnDiskInvertedLists jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, LongVector jarg5_, long jarg6); public final static native void OnDiskInvertedLists_resize(long jarg1, OnDiskInvertedLists jarg1_, long jarg2, long jarg3); public final static native long OnDiskInvertedLists_merge_from__SWIG_0(long jarg1, OnDiskInvertedLists jarg1_, long jarg2, int jarg3, boolean jarg4); public final static native long OnDiskInvertedLists_merge_from__SWIG_1(long jarg1, OnDiskInvertedLists jarg1_, long jarg2, int jarg3); public final static native long OnDiskInvertedLists_merge_from_1__SWIG_0(long jarg1, OnDiskInvertedLists jarg1_, long jarg2, InvertedLists jarg2_, boolean jarg3); public final static native long OnDiskInvertedLists_merge_from_1__SWIG_1(long jarg1, OnDiskInvertedLists jarg1_, long jarg2, InvertedLists jarg2_); public final static native void OnDiskInvertedLists_crop_invlists(long jarg1, OnDiskInvertedLists jarg1_, long jarg2, long jarg3); public final static native void OnDiskInvertedLists_prefetch_lists(long jarg1, OnDiskInvertedLists jarg1_, long jarg2, LongVector jarg2_, int jarg3); public final static native void delete_OnDiskInvertedLists(long jarg1); public final static native void OnDiskInvertedLists_locks_set(long jarg1, OnDiskInvertedLists jarg1_, long jarg2); public final static native long OnDiskInvertedLists_locks_get(long jarg1, OnDiskInvertedLists jarg1_); public final static native void OnDiskInvertedLists_pf_set(long jarg1, OnDiskInvertedLists jarg1_, long jarg2); public final static native long OnDiskInvertedLists_pf_get(long jarg1, OnDiskInvertedLists jarg1_); public final static native void OnDiskInvertedLists_prefetch_nthread_set(long jarg1, OnDiskInvertedLists jarg1_, int jarg2); public final static native int OnDiskInvertedLists_prefetch_nthread_get(long jarg1, OnDiskInvertedLists jarg1_); public final static native void OnDiskInvertedLists_do_mmap(long jarg1, OnDiskInvertedLists jarg1_); public final static native void OnDiskInvertedLists_update_totsize(long jarg1, OnDiskInvertedLists jarg1_, long jarg2); public final static native void OnDiskInvertedLists_resize_locked(long jarg1, OnDiskInvertedLists jarg1_, long jarg2, long jarg3); public final static native long OnDiskInvertedLists_allocate_slot(long jarg1, OnDiskInvertedLists jarg1_, long jarg2); public final static native void OnDiskInvertedLists_free_slot(long jarg1, OnDiskInvertedLists jarg1_, long jarg2, long jarg3); public final static native void OnDiskInvertedLists_set_all_lists_sizes(long jarg1, OnDiskInvertedLists jarg1_, long jarg2); public final static native long new_OnDiskInvertedLists__SWIG_1(); public final static native long new_OnDiskInvertedListsIOHook(); public final static native void OnDiskInvertedListsIOHook_write(long jarg1, OnDiskInvertedListsIOHook jarg1_, long jarg2, InvertedLists jarg2_, long jarg3); public final static native long OnDiskInvertedListsIOHook_read(long jarg1, OnDiskInvertedListsIOHook jarg1_, long jarg2, int jarg3); public final static native long OnDiskInvertedListsIOHook_read_ArrayInvertedLists(long jarg1, OnDiskInvertedListsIOHook jarg1_, long jarg2, int jarg3, long jarg4, long jarg5, long jarg6, Uint64Vector jarg6_); public final static native void delete_OnDiskInvertedListsIOHook(long jarg1); public final static native void IVFPQSearchParameters_scan_table_threshold_set(long jarg1, IVFPQSearchParameters jarg1_, long jarg2); public final static native long IVFPQSearchParameters_scan_table_threshold_get(long jarg1, IVFPQSearchParameters jarg1_); public final static native void IVFPQSearchParameters_polysemous_ht_set(long jarg1, IVFPQSearchParameters jarg1_, int jarg2); public final static native int IVFPQSearchParameters_polysemous_ht_get(long jarg1, IVFPQSearchParameters jarg1_); public final static native long new_IVFPQSearchParameters(); public final static native void delete_IVFPQSearchParameters(long jarg1); public final static native void precomputed_table_max_bytes_set(long jarg1); public final static native long precomputed_table_max_bytes_get(); public final static native void IndexIVFPQ_by_residual_set(long jarg1, IndexIVFPQ jarg1_, boolean jarg2); public final static native boolean IndexIVFPQ_by_residual_get(long jarg1, IndexIVFPQ jarg1_); public final static native void IndexIVFPQ_pq_set(long jarg1, IndexIVFPQ jarg1_, long jarg2, ProductQuantizer jarg2_); public final static native long IndexIVFPQ_pq_get(long jarg1, IndexIVFPQ jarg1_); public final static native void IndexIVFPQ_do_polysemous_training_set(long jarg1, IndexIVFPQ jarg1_, boolean jarg2); public final static native boolean IndexIVFPQ_do_polysemous_training_get(long jarg1, IndexIVFPQ jarg1_); public final static native void IndexIVFPQ_polysemous_training_set(long jarg1, IndexIVFPQ jarg1_, long jarg2, PolysemousTraining jarg2_); public final static native long IndexIVFPQ_polysemous_training_get(long jarg1, IndexIVFPQ jarg1_); public final static native void IndexIVFPQ_scan_table_threshold_set(long jarg1, IndexIVFPQ jarg1_, long jarg2); public final static native long IndexIVFPQ_scan_table_threshold_get(long jarg1, IndexIVFPQ jarg1_); public final static native void IndexIVFPQ_polysemous_ht_set(long jarg1, IndexIVFPQ jarg1_, int jarg2); public final static native int IndexIVFPQ_polysemous_ht_get(long jarg1, IndexIVFPQ jarg1_); public final static native void IndexIVFPQ_use_precomputed_table_set(long jarg1, IndexIVFPQ jarg1_, int jarg2); public final static native int IndexIVFPQ_use_precomputed_table_get(long jarg1, IndexIVFPQ jarg1_); public final static native void IndexIVFPQ_precomputed_table_set(long jarg1, IndexIVFPQ jarg1_, long jarg2); public final static native long IndexIVFPQ_precomputed_table_get(long jarg1, IndexIVFPQ jarg1_); public final static native long new_IndexIVFPQ__SWIG_0(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, int jarg6); public final static native long new_IndexIVFPQ__SWIG_1(long jarg1, Index jarg1_, long jarg2, long jarg3, long jarg4, long jarg5); public final static native void IndexIVFPQ_encode_vectors__SWIG_0(long jarg1, IndexIVFPQ jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, boolean jarg6); public final static native void IndexIVFPQ_encode_vectors__SWIG_1(long jarg1, IndexIVFPQ jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5); public final static native void IndexIVFPQ_sa_decode(long jarg1, IndexIVFPQ jarg1_, long jarg2, long jarg3, long jarg4); public final static native void IndexIVFPQ_add_core(long jarg1, IndexIVFPQ jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, LongVector jarg5_); public final static native void IndexIVFPQ_add_core_o__SWIG_0(long jarg1, IndexIVFPQ jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexIVFPQ_add_core_o__SWIG_1(long jarg1, IndexIVFPQ jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5); public final static native void IndexIVFPQ_train_residual(long jarg1, IndexIVFPQ jarg1_, long jarg2, long jarg3); public final static native void IndexIVFPQ_train_residual_o(long jarg1, IndexIVFPQ jarg1_, long jarg2, long jarg3, long jarg4); public final static native void IndexIVFPQ_reconstruct_from_offset(long jarg1, IndexIVFPQ jarg1_, long jarg2, long jarg3, long jarg4); public final static native long IndexIVFPQ_find_duplicates(long jarg1, IndexIVFPQ jarg1_, long jarg2, LongVector jarg2_, long jarg3); public final static native void IndexIVFPQ_encode(long jarg1, IndexIVFPQ jarg1_, long jarg2, long jarg3, long jarg4); public final static native void IndexIVFPQ_encode_multiple__SWIG_0(long jarg1, IndexIVFPQ jarg1_, long jarg2, long jarg3, LongVector jarg3_, long jarg4, long jarg5, boolean jarg6); public final static native void IndexIVFPQ_encode_multiple__SWIG_1(long jarg1, IndexIVFPQ jarg1_, long jarg2, long jarg3, LongVector jarg3_, long jarg4, long jarg5); public final static native void IndexIVFPQ_decode_multiple(long jarg1, IndexIVFPQ jarg1_, long jarg2, long jarg3, LongVector jarg3_, long jarg4, long jarg5); public final static native long IndexIVFPQ_get_InvertedListScanner(long jarg1, IndexIVFPQ jarg1_, boolean jarg2); public final static native void IndexIVFPQ_precompute_table(long jarg1, IndexIVFPQ jarg1_); public final static native long new_IndexIVFPQ__SWIG_2(); public final static native void delete_IndexIVFPQ(long jarg1); public final static native void initialize_IVFPQ_precomputed_table(long jarg1, long jarg2, Index jarg2_, long jarg3, ProductQuantizer jarg3_, long jarg4, boolean jarg5); public final static native void IndexIVFPQStats_nrefine_set(long jarg1, IndexIVFPQStats jarg1_, long jarg2); public final static native long IndexIVFPQStats_nrefine_get(long jarg1, IndexIVFPQStats jarg1_); public final static native void IndexIVFPQStats_n_hamming_pass_set(long jarg1, IndexIVFPQStats jarg1_, long jarg2); public final static native long IndexIVFPQStats_n_hamming_pass_get(long jarg1, IndexIVFPQStats jarg1_); public final static native void IndexIVFPQStats_search_cycles_set(long jarg1, IndexIVFPQStats jarg1_, long jarg2); public final static native long IndexIVFPQStats_search_cycles_get(long jarg1, IndexIVFPQStats jarg1_); public final static native void IndexIVFPQStats_refine_cycles_set(long jarg1, IndexIVFPQStats jarg1_, long jarg2); public final static native long IndexIVFPQStats_refine_cycles_get(long jarg1, IndexIVFPQStats jarg1_); public final static native long new_IndexIVFPQStats(); public final static native void IndexIVFPQStats_reset(long jarg1, IndexIVFPQStats jarg1_); public final static native void delete_IndexIVFPQStats(long jarg1); public final static native void indexIVFPQ_stats_set(long jarg1, IndexIVFPQStats jarg1_); public final static native long indexIVFPQ_stats_get(); public final static native void IndexBinary_d_set(long jarg1, IndexBinary jarg1_, int jarg2); public final static native int IndexBinary_d_get(long jarg1, IndexBinary jarg1_); public final static native void IndexBinary_code_size_set(long jarg1, IndexBinary jarg1_, int jarg2); public final static native int IndexBinary_code_size_get(long jarg1, IndexBinary jarg1_); public final static native void IndexBinary_ntotal_set(long jarg1, IndexBinary jarg1_, long jarg2); public final static native long IndexBinary_ntotal_get(long jarg1, IndexBinary jarg1_); public final static native void IndexBinary_verbose_set(long jarg1, IndexBinary jarg1_, boolean jarg2); public final static native boolean IndexBinary_verbose_get(long jarg1, IndexBinary jarg1_); public final static native void IndexBinary_is_trained_set(long jarg1, IndexBinary jarg1_, boolean jarg2); public final static native boolean IndexBinary_is_trained_get(long jarg1, IndexBinary jarg1_); public final static native void IndexBinary_metric_type_set(long jarg1, IndexBinary jarg1_, int jarg2); public final static native int IndexBinary_metric_type_get(long jarg1, IndexBinary jarg1_); public final static native void delete_IndexBinary(long jarg1); public final static native void IndexBinary_train(long jarg1, IndexBinary jarg1_, long jarg2, long jarg3); public final static native void IndexBinary_add(long jarg1, IndexBinary jarg1_, long jarg2, long jarg3); public final static native void IndexBinary_add_with_ids(long jarg1, IndexBinary jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_); public final static native void IndexBinary_search(long jarg1, IndexBinary jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexBinary_range_search(long jarg1, IndexBinary jarg1_, long jarg2, long jarg3, int jarg4, long jarg5, RangeSearchResult jarg5_); public final static native void IndexBinary_assign__SWIG_0(long jarg1, IndexBinary jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5); public final static native void IndexBinary_assign__SWIG_1(long jarg1, IndexBinary jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_); public final static native void IndexBinary_reset(long jarg1, IndexBinary jarg1_); public final static native long IndexBinary_remove_ids(long jarg1, IndexBinary jarg1_, long jarg2, IDSelector jarg2_); public final static native void IndexBinary_reconstruct(long jarg1, IndexBinary jarg1_, long jarg2, long jarg3); public final static native void IndexBinary_reconstruct_n(long jarg1, IndexBinary jarg1_, long jarg2, long jarg3, long jarg4); public final static native void IndexBinary_search_and_reconstruct(long jarg1, IndexBinary jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_, long jarg7); public final static native void IndexBinary_display(long jarg1, IndexBinary jarg1_); public final static native void Index2Layer_q1_set(long jarg1, Index2Layer jarg1_, long jarg2, Level1Quantizer jarg2_); public final static native long Index2Layer_q1_get(long jarg1, Index2Layer jarg1_); public final static native void Index2Layer_pq_set(long jarg1, Index2Layer jarg1_, long jarg2, ProductQuantizer jarg2_); public final static native long Index2Layer_pq_get(long jarg1, Index2Layer jarg1_); public final static native void Index2Layer_code_size_1_set(long jarg1, Index2Layer jarg1_, long jarg2); public final static native long Index2Layer_code_size_1_get(long jarg1, Index2Layer jarg1_); public final static native void Index2Layer_code_size_2_set(long jarg1, Index2Layer jarg1_, long jarg2); public final static native long Index2Layer_code_size_2_get(long jarg1, Index2Layer jarg1_); public final static native long new_Index2Layer__SWIG_0(long jarg1, Index jarg1_, long jarg2, int jarg3, int jarg4, int jarg5); public final static native long new_Index2Layer__SWIG_1(long jarg1, Index jarg1_, long jarg2, int jarg3, int jarg4); public final static native long new_Index2Layer__SWIG_2(long jarg1, Index jarg1_, long jarg2, int jarg3); public final static native long new_Index2Layer__SWIG_3(); public final static native void delete_Index2Layer(long jarg1); public final static native void Index2Layer_train(long jarg1, Index2Layer jarg1_, long jarg2, long jarg3); public final static native void Index2Layer_search(long jarg1, Index2Layer jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native long Index2Layer_get_distance_computer(long jarg1, Index2Layer jarg1_); public final static native void Index2Layer_transfer_to_IVFPQ(long jarg1, Index2Layer jarg1_, long jarg2, IndexIVFPQ jarg2_); public final static native void Index2Layer_sa_encode(long jarg1, Index2Layer jarg1_, long jarg2, long jarg3, long jarg4); public final static native void Index2Layer_sa_decode(long jarg1, Index2Layer jarg1_, long jarg2, long jarg3, long jarg4); public final static native void IndexBinaryFlat_xb_set(long jarg1, IndexBinaryFlat jarg1_, long jarg2, ByteVector jarg2_); public final static native long IndexBinaryFlat_xb_get(long jarg1, IndexBinaryFlat jarg1_); public final static native void IndexBinaryFlat_use_heap_set(long jarg1, IndexBinaryFlat jarg1_, boolean jarg2); public final static native boolean IndexBinaryFlat_use_heap_get(long jarg1, IndexBinaryFlat jarg1_); public final static native void IndexBinaryFlat_query_batch_size_set(long jarg1, IndexBinaryFlat jarg1_, long jarg2); public final static native long IndexBinaryFlat_query_batch_size_get(long jarg1, IndexBinaryFlat jarg1_); public final static native long new_IndexBinaryFlat__SWIG_0(long jarg1); public final static native void IndexBinaryFlat_add(long jarg1, IndexBinaryFlat jarg1_, long jarg2, long jarg3); public final static native void IndexBinaryFlat_reset(long jarg1, IndexBinaryFlat jarg1_); public final static native void IndexBinaryFlat_search(long jarg1, IndexBinaryFlat jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexBinaryFlat_range_search(long jarg1, IndexBinaryFlat jarg1_, long jarg2, long jarg3, int jarg4, long jarg5, RangeSearchResult jarg5_); public final static native void IndexBinaryFlat_reconstruct(long jarg1, IndexBinaryFlat jarg1_, long jarg2, long jarg3); public final static native long IndexBinaryFlat_remove_ids(long jarg1, IndexBinaryFlat jarg1_, long jarg2, IDSelector jarg2_); public final static native long new_IndexBinaryFlat__SWIG_1(); public final static native void delete_IndexBinaryFlat(long jarg1); public final static native void IndexBinaryIVF_invlists_set(long jarg1, IndexBinaryIVF jarg1_, long jarg2, InvertedLists jarg2_); public final static native long IndexBinaryIVF_invlists_get(long jarg1, IndexBinaryIVF jarg1_); public final static native void IndexBinaryIVF_own_invlists_set(long jarg1, IndexBinaryIVF jarg1_, boolean jarg2); public final static native boolean IndexBinaryIVF_own_invlists_get(long jarg1, IndexBinaryIVF jarg1_); public final static native void IndexBinaryIVF_nprobe_set(long jarg1, IndexBinaryIVF jarg1_, long jarg2); public final static native long IndexBinaryIVF_nprobe_get(long jarg1, IndexBinaryIVF jarg1_); public final static native void IndexBinaryIVF_max_codes_set(long jarg1, IndexBinaryIVF jarg1_, long jarg2); public final static native long IndexBinaryIVF_max_codes_get(long jarg1, IndexBinaryIVF jarg1_); public final static native void IndexBinaryIVF_use_heap_set(long jarg1, IndexBinaryIVF jarg1_, boolean jarg2); public final static native boolean IndexBinaryIVF_use_heap_get(long jarg1, IndexBinaryIVF jarg1_); public final static native void IndexBinaryIVF_direct_map_set(long jarg1, IndexBinaryIVF jarg1_, long jarg2); public final static native long IndexBinaryIVF_direct_map_get(long jarg1, IndexBinaryIVF jarg1_); public final static native void IndexBinaryIVF_quantizer_set(long jarg1, IndexBinaryIVF jarg1_, long jarg2, IndexBinary jarg2_); public final static native long IndexBinaryIVF_quantizer_get(long jarg1, IndexBinaryIVF jarg1_); public final static native void IndexBinaryIVF_nlist_set(long jarg1, IndexBinaryIVF jarg1_, long jarg2); public final static native long IndexBinaryIVF_nlist_get(long jarg1, IndexBinaryIVF jarg1_); public final static native void IndexBinaryIVF_own_fields_set(long jarg1, IndexBinaryIVF jarg1_, boolean jarg2); public final static native boolean IndexBinaryIVF_own_fields_get(long jarg1, IndexBinaryIVF jarg1_); public final static native void IndexBinaryIVF_cp_set(long jarg1, IndexBinaryIVF jarg1_, long jarg2, ClusteringParameters jarg2_); public final static native long IndexBinaryIVF_cp_get(long jarg1, IndexBinaryIVF jarg1_); public final static native void IndexBinaryIVF_clustering_index_set(long jarg1, IndexBinaryIVF jarg1_, long jarg2, Index jarg2_); public final static native long IndexBinaryIVF_clustering_index_get(long jarg1, IndexBinaryIVF jarg1_); public final static native long new_IndexBinaryIVF__SWIG_0(long jarg1, IndexBinary jarg1_, long jarg2, long jarg3); public final static native long new_IndexBinaryIVF__SWIG_1(); public final static native void delete_IndexBinaryIVF(long jarg1); public final static native void IndexBinaryIVF_reset(long jarg1, IndexBinaryIVF jarg1_); public final static native void IndexBinaryIVF_train(long jarg1, IndexBinaryIVF jarg1_, long jarg2, long jarg3); public final static native void IndexBinaryIVF_add(long jarg1, IndexBinaryIVF jarg1_, long jarg2, long jarg3); public final static native void IndexBinaryIVF_add_with_ids(long jarg1, IndexBinaryIVF jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_); public final static native void IndexBinaryIVF_add_core(long jarg1, IndexBinaryIVF jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, LongVector jarg5_); public final static native void IndexBinaryIVF_search_preassigned__SWIG_0(long jarg1, IndexBinaryIVF jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, LongVector jarg5_, long jarg6, long jarg7, long jarg8, LongVector jarg8_, boolean jarg9, long jarg10, IVFSearchParameters jarg10_); public final static native void IndexBinaryIVF_search_preassigned__SWIG_1(long jarg1, IndexBinaryIVF jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, LongVector jarg5_, long jarg6, long jarg7, long jarg8, LongVector jarg8_, boolean jarg9); public final static native long IndexBinaryIVF_get_InvertedListScanner__SWIG_0(long jarg1, IndexBinaryIVF jarg1_, boolean jarg2); public final static native long IndexBinaryIVF_get_InvertedListScanner__SWIG_1(long jarg1, IndexBinaryIVF jarg1_); public final static native void IndexBinaryIVF_search(long jarg1, IndexBinaryIVF jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexBinaryIVF_range_search(long jarg1, IndexBinaryIVF jarg1_, long jarg2, long jarg3, int jarg4, long jarg5, RangeSearchResult jarg5_); public final static native void IndexBinaryIVF_range_search_preassigned(long jarg1, IndexBinaryIVF jarg1_, long jarg2, long jarg3, int jarg4, long jarg5, LongVector jarg5_, long jarg6, long jarg7, RangeSearchResult jarg7_); public final static native void IndexBinaryIVF_reconstruct(long jarg1, IndexBinaryIVF jarg1_, long jarg2, long jarg3); public final static native void IndexBinaryIVF_reconstruct_n(long jarg1, IndexBinaryIVF jarg1_, long jarg2, long jarg3, long jarg4); public final static native void IndexBinaryIVF_search_and_reconstruct(long jarg1, IndexBinaryIVF jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_, long jarg7); public final static native void IndexBinaryIVF_reconstruct_from_offset(long jarg1, IndexBinaryIVF jarg1_, long jarg2, long jarg3, long jarg4); public final static native long IndexBinaryIVF_remove_ids(long jarg1, IndexBinaryIVF jarg1_, long jarg2, IDSelector jarg2_); public final static native void IndexBinaryIVF_merge_from(long jarg1, IndexBinaryIVF jarg1_, long jarg2, IndexBinaryIVF jarg2_, long jarg3); public final static native long IndexBinaryIVF_get_list_size(long jarg1, IndexBinaryIVF jarg1_, long jarg2); public final static native void IndexBinaryIVF_make_direct_map__SWIG_0(long jarg1, IndexBinaryIVF jarg1_, boolean jarg2); public final static native void IndexBinaryIVF_make_direct_map__SWIG_1(long jarg1, IndexBinaryIVF jarg1_); public final static native void IndexBinaryIVF_set_direct_map_type(long jarg1, IndexBinaryIVF jarg1_, long jarg2); public final static native void IndexBinaryIVF_replace_invlists__SWIG_0(long jarg1, IndexBinaryIVF jarg1_, long jarg2, InvertedLists jarg2_, boolean jarg3); public final static native void IndexBinaryIVF_replace_invlists__SWIG_1(long jarg1, IndexBinaryIVF jarg1_, long jarg2, InvertedLists jarg2_); public final static native void IndexBinaryFromFloat_index_set(long jarg1, IndexBinaryFromFloat jarg1_, long jarg2, Index jarg2_); public final static native long IndexBinaryFromFloat_index_get(long jarg1, IndexBinaryFromFloat jarg1_); public final static native void IndexBinaryFromFloat_own_fields_set(long jarg1, IndexBinaryFromFloat jarg1_, boolean jarg2); public final static native boolean IndexBinaryFromFloat_own_fields_get(long jarg1, IndexBinaryFromFloat jarg1_); public final static native long new_IndexBinaryFromFloat__SWIG_0(); public final static native long new_IndexBinaryFromFloat__SWIG_1(long jarg1, Index jarg1_); public final static native void delete_IndexBinaryFromFloat(long jarg1); public final static native void IndexBinaryFromFloat_add(long jarg1, IndexBinaryFromFloat jarg1_, long jarg2, long jarg3); public final static native void IndexBinaryFromFloat_reset(long jarg1, IndexBinaryFromFloat jarg1_); public final static native void IndexBinaryFromFloat_search(long jarg1, IndexBinaryFromFloat jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexBinaryFromFloat_train(long jarg1, IndexBinaryFromFloat jarg1_, long jarg2, long jarg3); public final static native void IndexBinaryHNSW_hnsw_set(long jarg1, IndexBinaryHNSW jarg1_, long jarg2, HNSW jarg2_); public final static native long IndexBinaryHNSW_hnsw_get(long jarg1, IndexBinaryHNSW jarg1_); public final static native void IndexBinaryHNSW_own_fields_set(long jarg1, IndexBinaryHNSW jarg1_, boolean jarg2); public final static native boolean IndexBinaryHNSW_own_fields_get(long jarg1, IndexBinaryHNSW jarg1_); public final static native void IndexBinaryHNSW_storage_set(long jarg1, IndexBinaryHNSW jarg1_, long jarg2, IndexBinary jarg2_); public final static native long IndexBinaryHNSW_storage_get(long jarg1, IndexBinaryHNSW jarg1_); public final static native long new_IndexBinaryHNSW__SWIG_0(); public final static native long new_IndexBinaryHNSW__SWIG_1(int jarg1, int jarg2); public final static native long new_IndexBinaryHNSW__SWIG_2(int jarg1); public final static native long new_IndexBinaryHNSW__SWIG_3(long jarg1, IndexBinary jarg1_, int jarg2); public final static native long new_IndexBinaryHNSW__SWIG_4(long jarg1, IndexBinary jarg1_); public final static native void delete_IndexBinaryHNSW(long jarg1); public final static native long IndexBinaryHNSW_get_distance_computer(long jarg1, IndexBinaryHNSW jarg1_); public final static native void IndexBinaryHNSW_add(long jarg1, IndexBinaryHNSW jarg1_, long jarg2, long jarg3); public final static native void IndexBinaryHNSW_train(long jarg1, IndexBinaryHNSW jarg1_, long jarg2, long jarg3); public final static native void IndexBinaryHNSW_search(long jarg1, IndexBinaryHNSW jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexBinaryHNSW_reconstruct(long jarg1, IndexBinaryHNSW jarg1_, long jarg2, long jarg3); public final static native void IndexBinaryHNSW_reset(long jarg1, IndexBinaryHNSW jarg1_); public final static native void IndexRefine_base_index_set(long jarg1, IndexRefine jarg1_, long jarg2, Index jarg2_); public final static native long IndexRefine_base_index_get(long jarg1, IndexRefine jarg1_); public final static native void IndexRefine_refine_index_set(long jarg1, IndexRefine jarg1_, long jarg2, Index jarg2_); public final static native long IndexRefine_refine_index_get(long jarg1, IndexRefine jarg1_); public final static native void IndexRefine_own_fields_set(long jarg1, IndexRefine jarg1_, boolean jarg2); public final static native boolean IndexRefine_own_fields_get(long jarg1, IndexRefine jarg1_); public final static native void IndexRefine_own_refine_index_set(long jarg1, IndexRefine jarg1_, boolean jarg2); public final static native boolean IndexRefine_own_refine_index_get(long jarg1, IndexRefine jarg1_); public final static native void IndexRefine_k_factor_set(long jarg1, IndexRefine jarg1_, float jarg2); public final static native float IndexRefine_k_factor_get(long jarg1, IndexRefine jarg1_); public final static native long new_IndexRefine__SWIG_0(long jarg1, Index jarg1_, long jarg2, Index jarg2_); public final static native long new_IndexRefine__SWIG_1(); public final static native void IndexRefine_train(long jarg1, IndexRefine jarg1_, long jarg2, long jarg3); public final static native void IndexRefine_add(long jarg1, IndexRefine jarg1_, long jarg2, long jarg3); public final static native void IndexRefine_reset(long jarg1, IndexRefine jarg1_); public final static native void IndexRefine_search(long jarg1, IndexRefine jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexRefine_reconstruct(long jarg1, IndexRefine jarg1_, long jarg2, long jarg3); public final static native long IndexRefine_sa_code_size(long jarg1, IndexRefine jarg1_); public final static native void IndexRefine_sa_encode(long jarg1, IndexRefine jarg1_, long jarg2, long jarg3, long jarg4); public final static native void IndexRefine_sa_decode(long jarg1, IndexRefine jarg1_, long jarg2, long jarg3, long jarg4); public final static native void delete_IndexRefine(long jarg1); public final static native long new_IndexRefineFlat__SWIG_0(long jarg1, Index jarg1_); public final static native long new_IndexRefineFlat__SWIG_1(long jarg1, Index jarg1_, long jarg2); public final static native long new_IndexRefineFlat__SWIG_2(); public final static native void IndexRefineFlat_search(long jarg1, IndexRefineFlat jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void delete_IndexRefineFlat(long jarg1); public final static native void IndexSplitVectors_own_fields_set(long jarg1, IndexSplitVectors jarg1_, boolean jarg2); public final static native boolean IndexSplitVectors_own_fields_get(long jarg1, IndexSplitVectors jarg1_); public final static native void IndexSplitVectors_threaded_set(long jarg1, IndexSplitVectors jarg1_, boolean jarg2); public final static native boolean IndexSplitVectors_threaded_get(long jarg1, IndexSplitVectors jarg1_); public final static native void IndexSplitVectors_sub_indexes_set(long jarg1, IndexSplitVectors jarg1_, long jarg2); public final static native long IndexSplitVectors_sub_indexes_get(long jarg1, IndexSplitVectors jarg1_); public final static native void IndexSplitVectors_sum_d_set(long jarg1, IndexSplitVectors jarg1_, long jarg2); public final static native long IndexSplitVectors_sum_d_get(long jarg1, IndexSplitVectors jarg1_); public final static native void IndexSplitVectors_add_sub_index(long jarg1, IndexSplitVectors jarg1_, long jarg2, Index jarg2_); public final static native void IndexSplitVectors_sync_with_sub_indexes(long jarg1, IndexSplitVectors jarg1_); public final static native void IndexSplitVectors_add(long jarg1, IndexSplitVectors jarg1_, long jarg2, long jarg3); public final static native void IndexSplitVectors_search(long jarg1, IndexSplitVectors jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexSplitVectors_train(long jarg1, IndexSplitVectors jarg1_, long jarg2, long jarg3); public final static native void IndexSplitVectors_reset(long jarg1, IndexSplitVectors jarg1_); public final static native void delete_IndexSplitVectors(long jarg1); public final static native void IndexIDMap_index_set(long jarg1, IndexIDMap jarg1_, long jarg2, Index jarg2_); public final static native long IndexIDMap_index_get(long jarg1, IndexIDMap jarg1_); public final static native void IndexIDMap_own_fields_set(long jarg1, IndexIDMap jarg1_, boolean jarg2); public final static native boolean IndexIDMap_own_fields_get(long jarg1, IndexIDMap jarg1_); public final static native void IndexIDMap_id_map_set(long jarg1, IndexIDMap jarg1_, long jarg2); public final static native long IndexIDMap_id_map_get(long jarg1, IndexIDMap jarg1_); public final static native long new_IndexIDMap__SWIG_0(long jarg1, Index jarg1_); public final static native void IndexIDMap_add_with_ids(long jarg1, IndexIDMap jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_); public final static native void IndexIDMap_add(long jarg1, IndexIDMap jarg1_, long jarg2, long jarg3); public final static native void IndexIDMap_search(long jarg1, IndexIDMap jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexIDMap_train(long jarg1, IndexIDMap jarg1_, long jarg2, long jarg3); public final static native void IndexIDMap_reset(long jarg1, IndexIDMap jarg1_); public final static native long IndexIDMap_remove_ids(long jarg1, IndexIDMap jarg1_, long jarg2, IDSelector jarg2_); public final static native void IndexIDMap_range_search(long jarg1, IndexIDMap jarg1_, long jarg2, long jarg3, float jarg4, long jarg5, RangeSearchResult jarg5_); public final static native void delete_IndexIDMap(long jarg1); public final static native long new_IndexIDMap__SWIG_1(); public final static native long new_IndexShards__SWIG_0(boolean jarg1, boolean jarg2); public final static native long new_IndexShards__SWIG_1(boolean jarg1); public final static native long new_IndexShards__SWIG_2(); public final static native long new_IndexShards__SWIG_3(int jarg1, boolean jarg2, boolean jarg3); public final static native long new_IndexShards__SWIG_4(int jarg1, boolean jarg2); public final static native long new_IndexShards__SWIG_5(int jarg1); public final static native void IndexShards_add_shard(long jarg1, IndexShards jarg1_, long jarg2, Index jarg2_); public final static native void IndexShards_remove_shard(long jarg1, IndexShards jarg1_, long jarg2, Index jarg2_); public final static native void IndexShards_add(long jarg1, IndexShards jarg1_, long jarg2, long jarg3); public final static native void IndexShards_add_with_ids(long jarg1, IndexShards jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_); public final static native void IndexShards_search(long jarg1, IndexShards jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6, LongVector jarg6_); public final static native void IndexShards_train(long jarg1, IndexShards jarg1_, long jarg2, long jarg3); public final static native void IndexShards_successive_ids_set(long jarg1, IndexShards jarg1_, boolean jarg2); public final static native boolean IndexShards_successive_ids_get(long jarg1, IndexShards jarg1_); public final static native void IndexShards_syncWithSubIndexes(long jarg1, IndexShards jarg1_); public final static native void delete_IndexShards(long jarg1); public final static native long downcast_index(long jarg1, Index jarg1_); public final static native long downcast_VectorTransform(long jarg1, VectorTransform jarg1_); public final static native long downcast_IndexBinary(long jarg1, IndexBinary jarg1_); public final static native long upcast_IndexShards(long jarg1, IndexShards jarg1_); public final static native void write_index__SWIG_0(long jarg1, Index jarg1_, String jarg2); public final static native void write_index__SWIG_1(long jarg1, Index jarg1_, long jarg2); public final static native void write_index__SWIG_2(long jarg1, Index jarg1_, long jarg2); public final static native void write_index_binary__SWIG_0(long jarg1, IndexBinary jarg1_, String jarg2); public final static native void write_index_binary__SWIG_1(long jarg1, IndexBinary jarg1_, long jarg2); public final static native void write_index_binary__SWIG_2(long jarg1, IndexBinary jarg1_, long jarg2); public final static native int IO_FLAG_READ_ONLY_get(); public final static native int IO_FLAG_ONDISK_SAME_DIR_get(); public final static native int IO_FLAG_SKIP_IVF_DATA_get(); public final static native int IO_FLAG_MMAP_get(); public final static native long read_index__SWIG_0(String jarg1, int jarg2); public final static native long read_index__SWIG_1(String jarg1); public final static native long read_index__SWIG_2(long jarg1, int jarg2); public final static native long read_index__SWIG_3(long jarg1); public final static native long read_index__SWIG_4(long jarg1, int jarg2); public final static native long read_index__SWIG_5(long jarg1); public final static native long read_index_binary__SWIG_0(String jarg1, int jarg2); public final static native long read_index_binary__SWIG_1(String jarg1); public final static native long read_index_binary__SWIG_2(long jarg1, int jarg2); public final static native long read_index_binary__SWIG_3(long jarg1); public final static native long read_index_binary__SWIG_4(long jarg1, int jarg2); public final static native long read_index_binary__SWIG_5(long jarg1); public final static native void write_VectorTransform(long jarg1, VectorTransform jarg1_, String jarg2); public final static native long read_VectorTransform(String jarg1); public final static native long read_ProductQuantizer__SWIG_0(String jarg1); public final static native long read_ProductQuantizer__SWIG_1(long jarg1); public final static native void write_ProductQuantizer__SWIG_0(long jarg1, ProductQuantizer jarg1_, String jarg2); public final static native void write_ProductQuantizer__SWIG_1(long jarg1, ProductQuantizer jarg1_, long jarg2); public final static native void write_InvertedLists(long jarg1, InvertedLists jarg1_, long jarg2); public final static native long read_InvertedLists__SWIG_0(long jarg1, int jarg2); public final static native long read_InvertedLists__SWIG_1(long jarg1); public final static native void AutoTuneCriterion_nq_set(long jarg1, AutoTuneCriterion jarg1_, long jarg2); public final static native long AutoTuneCriterion_nq_get(long jarg1, AutoTuneCriterion jarg1_); public final static native void AutoTuneCriterion_nnn_set(long jarg1, AutoTuneCriterion jarg1_, long jarg2); public final static native long AutoTuneCriterion_nnn_get(long jarg1, AutoTuneCriterion jarg1_); public final static native void AutoTuneCriterion_gt_nnn_set(long jarg1, AutoTuneCriterion jarg1_, long jarg2); public final static native long AutoTuneCriterion_gt_nnn_get(long jarg1, AutoTuneCriterion jarg1_); public final static native void AutoTuneCriterion_gt_D_set(long jarg1, AutoTuneCriterion jarg1_, long jarg2, FloatVector jarg2_); public final static native long AutoTuneCriterion_gt_D_get(long jarg1, AutoTuneCriterion jarg1_); public final static native void AutoTuneCriterion_gt_I_set(long jarg1, AutoTuneCriterion jarg1_, long jarg2); public final static native long AutoTuneCriterion_gt_I_get(long jarg1, AutoTuneCriterion jarg1_); public final static native void AutoTuneCriterion_set_groundtruth(long jarg1, AutoTuneCriterion jarg1_, int jarg2, long jarg3, long jarg4, LongVector jarg4_); public final static native double AutoTuneCriterion_evaluate(long jarg1, AutoTuneCriterion jarg1_, long jarg2, long jarg3, LongVector jarg3_); public final static native void delete_AutoTuneCriterion(long jarg1); public final static native void OneRecallAtRCriterion_R_set(long jarg1, OneRecallAtRCriterion jarg1_, long jarg2); public final static native long OneRecallAtRCriterion_R_get(long jarg1, OneRecallAtRCriterion jarg1_); public final static native long new_OneRecallAtRCriterion(long jarg1, long jarg2); public final static native double OneRecallAtRCriterion_evaluate(long jarg1, OneRecallAtRCriterion jarg1_, long jarg2, long jarg3, LongVector jarg3_); public final static native void delete_OneRecallAtRCriterion(long jarg1); public final static native void IntersectionCriterion_R_set(long jarg1, IntersectionCriterion jarg1_, long jarg2); public final static native long IntersectionCriterion_R_get(long jarg1, IntersectionCriterion jarg1_); public final static native long new_IntersectionCriterion(long jarg1, long jarg2); public final static native double IntersectionCriterion_evaluate(long jarg1, IntersectionCriterion jarg1_, long jarg2, long jarg3, LongVector jarg3_); public final static native void delete_IntersectionCriterion(long jarg1); public final static native void OperatingPoint_perf_set(long jarg1, OperatingPoint jarg1_, double jarg2); public final static native double OperatingPoint_perf_get(long jarg1, OperatingPoint jarg1_); public final static native void OperatingPoint_t_set(long jarg1, OperatingPoint jarg1_, double jarg2); public final static native double OperatingPoint_t_get(long jarg1, OperatingPoint jarg1_); public final static native void OperatingPoint_key_set(long jarg1, OperatingPoint jarg1_, String jarg2); public final static native String OperatingPoint_key_get(long jarg1, OperatingPoint jarg1_); public final static native void OperatingPoint_cno_set(long jarg1, OperatingPoint jarg1_, long jarg2); public final static native long OperatingPoint_cno_get(long jarg1, OperatingPoint jarg1_); public final static native long new_OperatingPoint(); public final static native void delete_OperatingPoint(long jarg1); public final static native void OperatingPoints_all_pts_set(long jarg1, OperatingPoints jarg1_, long jarg2, OperatingPointVector jarg2_); public final static native long OperatingPoints_all_pts_get(long jarg1, OperatingPoints jarg1_); public final static native void OperatingPoints_optimal_pts_set(long jarg1, OperatingPoints jarg1_, long jarg2, OperatingPointVector jarg2_); public final static native long OperatingPoints_optimal_pts_get(long jarg1, OperatingPoints jarg1_); public final static native long new_OperatingPoints(); public final static native int OperatingPoints_merge_with__SWIG_0(long jarg1, OperatingPoints jarg1_, long jarg2, OperatingPoints jarg2_, String jarg3); public final static native int OperatingPoints_merge_with__SWIG_1(long jarg1, OperatingPoints jarg1_, long jarg2, OperatingPoints jarg2_); public final static native void OperatingPoints_clear(long jarg1, OperatingPoints jarg1_); public final static native boolean OperatingPoints_add__SWIG_0(long jarg1, OperatingPoints jarg1_, double jarg2, double jarg3, String jarg4, long jarg5); public final static native boolean OperatingPoints_add__SWIG_1(long jarg1, OperatingPoints jarg1_, double jarg2, double jarg3, String jarg4); public final static native double OperatingPoints_t_for_perf(long jarg1, OperatingPoints jarg1_, double jarg2); public final static native void OperatingPoints_display__SWIG_0(long jarg1, OperatingPoints jarg1_, boolean jarg2); public final static native void OperatingPoints_display__SWIG_1(long jarg1, OperatingPoints jarg1_); public final static native void OperatingPoints_all_to_gnuplot(long jarg1, OperatingPoints jarg1_, String jarg2); public final static native void OperatingPoints_optimal_to_gnuplot(long jarg1, OperatingPoints jarg1_, String jarg2); public final static native void delete_OperatingPoints(long jarg1); public final static native void ParameterRange_name_set(long jarg1, ParameterRange jarg1_, String jarg2); public final static native String ParameterRange_name_get(long jarg1, ParameterRange jarg1_); public final static native void ParameterRange_values_set(long jarg1, ParameterRange jarg1_, long jarg2, DoubleVector jarg2_); public final static native long ParameterRange_values_get(long jarg1, ParameterRange jarg1_); public final static native long new_ParameterRange(); public final static native void delete_ParameterRange(long jarg1); public final static native void ParameterSpace_parameter_ranges_set(long jarg1, ParameterSpace jarg1_, long jarg2); public final static native long ParameterSpace_parameter_ranges_get(long jarg1, ParameterSpace jarg1_); public final static native void ParameterSpace_verbose_set(long jarg1, ParameterSpace jarg1_, int jarg2); public final static native int ParameterSpace_verbose_get(long jarg1, ParameterSpace jarg1_); public final static native void ParameterSpace_n_experiments_set(long jarg1, ParameterSpace jarg1_, int jarg2); public final static native int ParameterSpace_n_experiments_get(long jarg1, ParameterSpace jarg1_); public final static native void ParameterSpace_batchsize_set(long jarg1, ParameterSpace jarg1_, long jarg2); public final static native long ParameterSpace_batchsize_get(long jarg1, ParameterSpace jarg1_); public final static native void ParameterSpace_thread_over_batches_set(long jarg1, ParameterSpace jarg1_, boolean jarg2); public final static native boolean ParameterSpace_thread_over_batches_get(long jarg1, ParameterSpace jarg1_); public final static native void ParameterSpace_min_test_duration_set(long jarg1, ParameterSpace jarg1_, double jarg2); public final static native double ParameterSpace_min_test_duration_get(long jarg1, ParameterSpace jarg1_); public final static native long new_ParameterSpace(); public final static native long ParameterSpace_n_combinations(long jarg1, ParameterSpace jarg1_); public final static native boolean ParameterSpace_combination_ge(long jarg1, ParameterSpace jarg1_, long jarg2, long jarg3); public final static native String ParameterSpace_combination_name(long jarg1, ParameterSpace jarg1_, long jarg2); public final static native void ParameterSpace_display(long jarg1, ParameterSpace jarg1_); public final static native long ParameterSpace_add_range(long jarg1, ParameterSpace jarg1_, String jarg2); public final static native void ParameterSpace_initialize(long jarg1, ParameterSpace jarg1_, long jarg2, Index jarg2_); public final static native void ParameterSpace_set_index_parameters__SWIG_0(long jarg1, ParameterSpace jarg1_, long jarg2, Index jarg2_, long jarg3); public final static native void ParameterSpace_set_index_parameters__SWIG_1(long jarg1, ParameterSpace jarg1_, long jarg2, Index jarg2_, String jarg3); public final static native void ParameterSpace_set_index_parameter(long jarg1, ParameterSpace jarg1_, long jarg2, Index jarg2_, String jarg3, double jarg4); public final static native void ParameterSpace_update_bounds(long jarg1, ParameterSpace jarg1_, long jarg2, long jarg3, OperatingPoint jarg3_, long jarg4, long jarg5); public final static native void ParameterSpace_explore(long jarg1, ParameterSpace jarg1_, long jarg2, Index jarg2_, long jarg3, long jarg4, long jarg5, AutoTuneCriterion jarg5_, long jarg6, OperatingPoints jarg6_); public final static native void delete_ParameterSpace(long jarg1); public final static native long index_factory__SWIG_0(int jarg1, String jarg2, int jarg3); public final static native long index_factory__SWIG_1(int jarg1, String jarg2); public final static native void index_factory_verbose_set(int jarg1); public final static native int index_factory_verbose_get(); public final static native long index_binary_factory(int jarg1, String jarg2); public final static native void simd_histogram_8(long jarg1, int jarg2, long jarg3, int jarg4, long jarg5); public final static native void simd_histogram_16(long jarg1, int jarg2, long jarg3, int jarg4, long jarg5); public final static native void PartitionStats_bissect_cycles_set(long jarg1, PartitionStats jarg1_, long jarg2); public final static native long PartitionStats_bissect_cycles_get(long jarg1, PartitionStats jarg1_); public final static native void PartitionStats_compress_cycles_set(long jarg1, PartitionStats jarg1_, long jarg2); public final static native long PartitionStats_compress_cycles_get(long jarg1, PartitionStats jarg1_); public final static native long new_PartitionStats(); public final static native void PartitionStats_reset(long jarg1, PartitionStats jarg1_); public final static native void delete_PartitionStats(long jarg1); public final static native void partition_stats_set(long jarg1, PartitionStats jarg1_); public final static native long partition_stats_get(); public final static native void float_minheap_array_t_nh_set(long jarg1, float_minheap_array_t jarg1_, long jarg2); public final static native long float_minheap_array_t_nh_get(long jarg1, float_minheap_array_t jarg1_); public final static native void float_minheap_array_t_k_set(long jarg1, float_minheap_array_t jarg1_, long jarg2); public final static native long float_minheap_array_t_k_get(long jarg1, float_minheap_array_t jarg1_); public final static native void float_minheap_array_t_ids_set(long jarg1, float_minheap_array_t jarg1_, long jarg2, LongVector jarg2_); public final static native long float_minheap_array_t_ids_get(long jarg1, float_minheap_array_t jarg1_); public final static native void float_minheap_array_t_val_set(long jarg1, float_minheap_array_t jarg1_, long jarg2); public final static native long float_minheap_array_t_val_get(long jarg1, float_minheap_array_t jarg1_); public final static native long float_minheap_array_t_get_val(long jarg1, float_minheap_array_t jarg1_, long jarg2); public final static native long float_minheap_array_t_get_ids(long jarg1, float_minheap_array_t jarg1_, long jarg2); public final static native void float_minheap_array_t_heapify(long jarg1, float_minheap_array_t jarg1_); public final static native void float_minheap_array_t_addn__SWIG_0(long jarg1, float_minheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6); public final static native void float_minheap_array_t_addn__SWIG_1(long jarg1, float_minheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, long jarg5); public final static native void float_minheap_array_t_addn__SWIG_2(long jarg1, float_minheap_array_t jarg1_, long jarg2, long jarg3, long jarg4); public final static native void float_minheap_array_t_addn__SWIG_3(long jarg1, float_minheap_array_t jarg1_, long jarg2, long jarg3); public final static native void float_minheap_array_t_addn_with_ids__SWIG_0(long jarg1, float_minheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, long jarg6, long jarg7); public final static native void float_minheap_array_t_addn_with_ids__SWIG_1(long jarg1, float_minheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, long jarg6); public final static native void float_minheap_array_t_addn_with_ids__SWIG_2(long jarg1, float_minheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5); public final static native void float_minheap_array_t_addn_with_ids__SWIG_3(long jarg1, float_minheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_); public final static native void float_minheap_array_t_addn_with_ids__SWIG_4(long jarg1, float_minheap_array_t jarg1_, long jarg2, long jarg3); public final static native void float_minheap_array_t_reorder(long jarg1, float_minheap_array_t jarg1_); public final static native void float_minheap_array_t_per_line_extrema(long jarg1, float_minheap_array_t jarg1_, long jarg2, long jarg3, LongVector jarg3_); public final static native long new_float_minheap_array_t(); public final static native void delete_float_minheap_array_t(long jarg1); public final static native void int_minheap_array_t_nh_set(long jarg1, int_minheap_array_t jarg1_, long jarg2); public final static native long int_minheap_array_t_nh_get(long jarg1, int_minheap_array_t jarg1_); public final static native void int_minheap_array_t_k_set(long jarg1, int_minheap_array_t jarg1_, long jarg2); public final static native long int_minheap_array_t_k_get(long jarg1, int_minheap_array_t jarg1_); public final static native void int_minheap_array_t_ids_set(long jarg1, int_minheap_array_t jarg1_, long jarg2, LongVector jarg2_); public final static native long int_minheap_array_t_ids_get(long jarg1, int_minheap_array_t jarg1_); public final static native void int_minheap_array_t_val_set(long jarg1, int_minheap_array_t jarg1_, long jarg2); public final static native long int_minheap_array_t_val_get(long jarg1, int_minheap_array_t jarg1_); public final static native long int_minheap_array_t_get_val(long jarg1, int_minheap_array_t jarg1_, long jarg2); public final static native long int_minheap_array_t_get_ids(long jarg1, int_minheap_array_t jarg1_, long jarg2); public final static native void int_minheap_array_t_heapify(long jarg1, int_minheap_array_t jarg1_); public final static native void int_minheap_array_t_addn__SWIG_0(long jarg1, int_minheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6); public final static native void int_minheap_array_t_addn__SWIG_1(long jarg1, int_minheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, long jarg5); public final static native void int_minheap_array_t_addn__SWIG_2(long jarg1, int_minheap_array_t jarg1_, long jarg2, long jarg3, long jarg4); public final static native void int_minheap_array_t_addn__SWIG_3(long jarg1, int_minheap_array_t jarg1_, long jarg2, long jarg3); public final static native void int_minheap_array_t_addn_with_ids__SWIG_0(long jarg1, int_minheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, long jarg6, long jarg7); public final static native void int_minheap_array_t_addn_with_ids__SWIG_1(long jarg1, int_minheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, long jarg6); public final static native void int_minheap_array_t_addn_with_ids__SWIG_2(long jarg1, int_minheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5); public final static native void int_minheap_array_t_addn_with_ids__SWIG_3(long jarg1, int_minheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_); public final static native void int_minheap_array_t_addn_with_ids__SWIG_4(long jarg1, int_minheap_array_t jarg1_, long jarg2, long jarg3); public final static native void int_minheap_array_t_reorder(long jarg1, int_minheap_array_t jarg1_); public final static native void int_minheap_array_t_per_line_extrema(long jarg1, int_minheap_array_t jarg1_, long jarg2, long jarg3, LongVector jarg3_); public final static native long new_int_minheap_array_t(); public final static native void delete_int_minheap_array_t(long jarg1); public final static native void float_maxheap_array_t_nh_set(long jarg1, float_maxheap_array_t jarg1_, long jarg2); public final static native long float_maxheap_array_t_nh_get(long jarg1, float_maxheap_array_t jarg1_); public final static native void float_maxheap_array_t_k_set(long jarg1, float_maxheap_array_t jarg1_, long jarg2); public final static native long float_maxheap_array_t_k_get(long jarg1, float_maxheap_array_t jarg1_); public final static native void float_maxheap_array_t_ids_set(long jarg1, float_maxheap_array_t jarg1_, long jarg2, LongVector jarg2_); public final static native long float_maxheap_array_t_ids_get(long jarg1, float_maxheap_array_t jarg1_); public final static native void float_maxheap_array_t_val_set(long jarg1, float_maxheap_array_t jarg1_, long jarg2); public final static native long float_maxheap_array_t_val_get(long jarg1, float_maxheap_array_t jarg1_); public final static native long float_maxheap_array_t_get_val(long jarg1, float_maxheap_array_t jarg1_, long jarg2); public final static native long float_maxheap_array_t_get_ids(long jarg1, float_maxheap_array_t jarg1_, long jarg2); public final static native void float_maxheap_array_t_heapify(long jarg1, float_maxheap_array_t jarg1_); public final static native void float_maxheap_array_t_addn__SWIG_0(long jarg1, float_maxheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6); public final static native void float_maxheap_array_t_addn__SWIG_1(long jarg1, float_maxheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, long jarg5); public final static native void float_maxheap_array_t_addn__SWIG_2(long jarg1, float_maxheap_array_t jarg1_, long jarg2, long jarg3, long jarg4); public final static native void float_maxheap_array_t_addn__SWIG_3(long jarg1, float_maxheap_array_t jarg1_, long jarg2, long jarg3); public final static native void float_maxheap_array_t_addn_with_ids__SWIG_0(long jarg1, float_maxheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, long jarg6, long jarg7); public final static native void float_maxheap_array_t_addn_with_ids__SWIG_1(long jarg1, float_maxheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, long jarg6); public final static native void float_maxheap_array_t_addn_with_ids__SWIG_2(long jarg1, float_maxheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5); public final static native void float_maxheap_array_t_addn_with_ids__SWIG_3(long jarg1, float_maxheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_); public final static native void float_maxheap_array_t_addn_with_ids__SWIG_4(long jarg1, float_maxheap_array_t jarg1_, long jarg2, long jarg3); public final static native void float_maxheap_array_t_reorder(long jarg1, float_maxheap_array_t jarg1_); public final static native void float_maxheap_array_t_per_line_extrema(long jarg1, float_maxheap_array_t jarg1_, long jarg2, long jarg3, LongVector jarg3_); public final static native long new_float_maxheap_array_t(); public final static native void delete_float_maxheap_array_t(long jarg1); public final static native void int_maxheap_array_t_nh_set(long jarg1, int_maxheap_array_t jarg1_, long jarg2); public final static native long int_maxheap_array_t_nh_get(long jarg1, int_maxheap_array_t jarg1_); public final static native void int_maxheap_array_t_k_set(long jarg1, int_maxheap_array_t jarg1_, long jarg2); public final static native long int_maxheap_array_t_k_get(long jarg1, int_maxheap_array_t jarg1_); public final static native void int_maxheap_array_t_ids_set(long jarg1, int_maxheap_array_t jarg1_, long jarg2, LongVector jarg2_); public final static native long int_maxheap_array_t_ids_get(long jarg1, int_maxheap_array_t jarg1_); public final static native void int_maxheap_array_t_val_set(long jarg1, int_maxheap_array_t jarg1_, long jarg2); public final static native long int_maxheap_array_t_val_get(long jarg1, int_maxheap_array_t jarg1_); public final static native long int_maxheap_array_t_get_val(long jarg1, int_maxheap_array_t jarg1_, long jarg2); public final static native long int_maxheap_array_t_get_ids(long jarg1, int_maxheap_array_t jarg1_, long jarg2); public final static native void int_maxheap_array_t_heapify(long jarg1, int_maxheap_array_t jarg1_); public final static native void int_maxheap_array_t_addn__SWIG_0(long jarg1, int_maxheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6); public final static native void int_maxheap_array_t_addn__SWIG_1(long jarg1, int_maxheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, long jarg5); public final static native void int_maxheap_array_t_addn__SWIG_2(long jarg1, int_maxheap_array_t jarg1_, long jarg2, long jarg3, long jarg4); public final static native void int_maxheap_array_t_addn__SWIG_3(long jarg1, int_maxheap_array_t jarg1_, long jarg2, long jarg3); public final static native void int_maxheap_array_t_addn_with_ids__SWIG_0(long jarg1, int_maxheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, long jarg6, long jarg7); public final static native void int_maxheap_array_t_addn_with_ids__SWIG_1(long jarg1, int_maxheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5, long jarg6); public final static native void int_maxheap_array_t_addn_with_ids__SWIG_2(long jarg1, int_maxheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5); public final static native void int_maxheap_array_t_addn_with_ids__SWIG_3(long jarg1, int_maxheap_array_t jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_); public final static native void int_maxheap_array_t_addn_with_ids__SWIG_4(long jarg1, int_maxheap_array_t jarg1_, long jarg2, long jarg3); public final static native void int_maxheap_array_t_reorder(long jarg1, int_maxheap_array_t jarg1_); public final static native void int_maxheap_array_t_per_line_extrema(long jarg1, int_maxheap_array_t jarg1_, long jarg2, long jarg3, LongVector jarg3_); public final static native long new_int_maxheap_array_t(); public final static native void delete_int_maxheap_array_t(long jarg1); public final static native float CMin_float_partition_fuzzy(long jarg1, long jarg2, LongVector jarg2_, long jarg3, long jarg4, long jarg5, long jarg6); public final static native float CMax_float_partition_fuzzy(long jarg1, long jarg2, LongVector jarg2_, long jarg3, long jarg4, long jarg5, long jarg6); public final static native void AlignedTableUint8_tab_set(long jarg1, AlignedTableUint8 jarg1_, long jarg2); public final static native long AlignedTableUint8_tab_get(long jarg1, AlignedTableUint8 jarg1_); public final static native void AlignedTableUint8_numel_set(long jarg1, AlignedTableUint8 jarg1_, long jarg2); public final static native long AlignedTableUint8_numel_get(long jarg1, AlignedTableUint8 jarg1_); public final static native long AlignedTableUint8_round_capacity(long jarg1); public final static native long new_AlignedTableUint8__SWIG_0(); public final static native long new_AlignedTableUint8__SWIG_1(long jarg1); public final static native long AlignedTableUint8_itemsize(long jarg1, AlignedTableUint8 jarg1_); public final static native void AlignedTableUint8_resize(long jarg1, AlignedTableUint8 jarg1_, long jarg2); public final static native void AlignedTableUint8_clear(long jarg1, AlignedTableUint8 jarg1_); public final static native long AlignedTableUint8_size(long jarg1, AlignedTableUint8 jarg1_); public final static native long AlignedTableUint8_nbytes(long jarg1, AlignedTableUint8 jarg1_); public final static native long AlignedTableUint8_get__SWIG_0(long jarg1, AlignedTableUint8 jarg1_); public final static native long AlignedTableUint8_data__SWIG_0(long jarg1, AlignedTableUint8 jarg1_); public final static native void delete_AlignedTableUint8(long jarg1); public final static native void AlignedTableUint16_tab_set(long jarg1, AlignedTableUint16 jarg1_, long jarg2); public final static native long AlignedTableUint16_tab_get(long jarg1, AlignedTableUint16 jarg1_); public final static native void AlignedTableUint16_numel_set(long jarg1, AlignedTableUint16 jarg1_, long jarg2); public final static native long AlignedTableUint16_numel_get(long jarg1, AlignedTableUint16 jarg1_); public final static native long AlignedTableUint16_round_capacity(long jarg1); public final static native long new_AlignedTableUint16__SWIG_0(); public final static native long new_AlignedTableUint16__SWIG_1(long jarg1); public final static native long AlignedTableUint16_itemsize(long jarg1, AlignedTableUint16 jarg1_); public final static native void AlignedTableUint16_resize(long jarg1, AlignedTableUint16 jarg1_, long jarg2); public final static native void AlignedTableUint16_clear(long jarg1, AlignedTableUint16 jarg1_); public final static native long AlignedTableUint16_size(long jarg1, AlignedTableUint16 jarg1_); public final static native long AlignedTableUint16_nbytes(long jarg1, AlignedTableUint16 jarg1_); public final static native long AlignedTableUint16_get__SWIG_0(long jarg1, AlignedTableUint16 jarg1_); public final static native long AlignedTableUint16_data__SWIG_0(long jarg1, AlignedTableUint16 jarg1_); public final static native void delete_AlignedTableUint16(long jarg1); public final static native void AlignedTableFloat32_tab_set(long jarg1, AlignedTableFloat32 jarg1_, long jarg2); public final static native long AlignedTableFloat32_tab_get(long jarg1, AlignedTableFloat32 jarg1_); public final static native void AlignedTableFloat32_numel_set(long jarg1, AlignedTableFloat32 jarg1_, long jarg2); public final static native long AlignedTableFloat32_numel_get(long jarg1, AlignedTableFloat32 jarg1_); public final static native long AlignedTableFloat32_round_capacity(long jarg1); public final static native long new_AlignedTableFloat32__SWIG_0(); public final static native long new_AlignedTableFloat32__SWIG_1(long jarg1); public final static native long AlignedTableFloat32_itemsize(long jarg1, AlignedTableFloat32 jarg1_); public final static native void AlignedTableFloat32_resize(long jarg1, AlignedTableFloat32 jarg1_, long jarg2); public final static native void AlignedTableFloat32_clear(long jarg1, AlignedTableFloat32 jarg1_); public final static native long AlignedTableFloat32_size(long jarg1, AlignedTableFloat32 jarg1_); public final static native long AlignedTableFloat32_nbytes(long jarg1, AlignedTableFloat32 jarg1_); public final static native long AlignedTableFloat32_get__SWIG_0(long jarg1, AlignedTableFloat32 jarg1_); public final static native long AlignedTableFloat32_data__SWIG_0(long jarg1, AlignedTableFloat32 jarg1_); public final static native void delete_AlignedTableFloat32(long jarg1); public final static native long CMax_uint16_partition_fuzzy__SWIG_0(long jarg1, long jarg2, LongVector jarg2_, long jarg3, long jarg4, long jarg5, long jarg6); public final static native long CMin_uint16_partition_fuzzy__SWIG_0(long jarg1, long jarg2, LongVector jarg2_, long jarg3, long jarg4, long jarg5, long jarg6); public final static native long CMax_uint16_partition_fuzzy__SWIG_1(long jarg1, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6); public final static native long CMin_uint16_partition_fuzzy__SWIG_1(long jarg1, long jarg2, long jarg3, long jarg4, long jarg5, long jarg6); public final static native void omp_set_num_threads(int jarg1); public final static native int omp_get_max_threads(); public final static native long memcpy(long jarg1, long jarg2, long jarg3); public final static native long cast_integer_to_float_ptr(int jarg1); public final static native long cast_integer_to_long_ptr(int jarg1); public final static native long cast_integer_to_int_ptr(int jarg1); public final static native void RangeSearchResult_nq_set(long jarg1, RangeSearchResult jarg1_, long jarg2); public final static native long RangeSearchResult_nq_get(long jarg1, RangeSearchResult jarg1_); public final static native void RangeSearchResult_lims_set(long jarg1, RangeSearchResult jarg1_, long jarg2); public final static native long RangeSearchResult_lims_get(long jarg1, RangeSearchResult jarg1_); public final static native void RangeSearchResult_labels_set(long jarg1, RangeSearchResult jarg1_, long jarg2, LongVector jarg2_); public final static native long RangeSearchResult_labels_get(long jarg1, RangeSearchResult jarg1_); public final static native void RangeSearchResult_distances_set(long jarg1, RangeSearchResult jarg1_, long jarg2); public final static native long RangeSearchResult_distances_get(long jarg1, RangeSearchResult jarg1_); public final static native void RangeSearchResult_buffer_size_set(long jarg1, RangeSearchResult jarg1_, long jarg2); public final static native long RangeSearchResult_buffer_size_get(long jarg1, RangeSearchResult jarg1_); public final static native void RangeSearchResult_do_allocation(long jarg1, RangeSearchResult jarg1_); public final static native void delete_RangeSearchResult(long jarg1); public final static native boolean IDSelector_is_member(long jarg1, IDSelector jarg1_, long jarg2); public final static native void delete_IDSelector(long jarg1); public final static native void IDSelectorRange_imin_set(long jarg1, IDSelectorRange jarg1_, long jarg2); public final static native long IDSelectorRange_imin_get(long jarg1, IDSelectorRange jarg1_); public final static native void IDSelectorRange_imax_set(long jarg1, IDSelectorRange jarg1_, long jarg2); public final static native long IDSelectorRange_imax_get(long jarg1, IDSelectorRange jarg1_); public final static native long new_IDSelectorRange(long jarg1, long jarg2); public final static native boolean IDSelectorRange_is_member(long jarg1, IDSelectorRange jarg1_, long jarg2); public final static native void delete_IDSelectorRange(long jarg1); public final static native void IDSelectorArray_n_set(long jarg1, IDSelectorArray jarg1_, long jarg2); public final static native long IDSelectorArray_n_get(long jarg1, IDSelectorArray jarg1_); public final static native void IDSelectorArray_ids_set(long jarg1, IDSelectorArray jarg1_, long jarg2, LongVector jarg2_); public final static native long IDSelectorArray_ids_get(long jarg1, IDSelectorArray jarg1_); public final static native long new_IDSelectorArray(long jarg1, long jarg2, LongVector jarg2_); public final static native boolean IDSelectorArray_is_member(long jarg1, IDSelectorArray jarg1_, long jarg2); public final static native void delete_IDSelectorArray(long jarg1); public final static native void IDSelectorBatch_nbits_set(long jarg1, IDSelectorBatch jarg1_, int jarg2); public final static native int IDSelectorBatch_nbits_get(long jarg1, IDSelectorBatch jarg1_); public final static native void IDSelectorBatch_mask_set(long jarg1, IDSelectorBatch jarg1_, long jarg2); public final static native long IDSelectorBatch_mask_get(long jarg1, IDSelectorBatch jarg1_); public final static native long new_IDSelectorBatch(long jarg1, long jarg2, LongVector jarg2_); public final static native boolean IDSelectorBatch_is_member(long jarg1, IDSelectorBatch jarg1_, long jarg2); public final static native void delete_IDSelectorBatch(long jarg1); public final static native void BufferList_buffer_size_set(long jarg1, BufferList jarg1_, long jarg2); public final static native long BufferList_buffer_size_get(long jarg1, BufferList jarg1_); public final static native void BufferList_buffers_set(long jarg1, BufferList jarg1_, long jarg2); public final static native long BufferList_buffers_get(long jarg1, BufferList jarg1_); public final static native void BufferList_wp_set(long jarg1, BufferList jarg1_, long jarg2); public final static native long BufferList_wp_get(long jarg1, BufferList jarg1_); public final static native long new_BufferList(long jarg1); public final static native void delete_BufferList(long jarg1); public final static native void BufferList_append_buffer(long jarg1, BufferList jarg1_); public final static native void BufferList_add(long jarg1, BufferList jarg1_, long jarg2, float jarg3); public final static native void BufferList_copy_range(long jarg1, BufferList jarg1_, long jarg2, long jarg3, long jarg4, LongVector jarg4_, long jarg5); public final static native void RangeQueryResult_qno_set(long jarg1, RangeQueryResult jarg1_, long jarg2); public final static native long RangeQueryResult_qno_get(long jarg1, RangeQueryResult jarg1_); public final static native void RangeQueryResult_nres_set(long jarg1, RangeQueryResult jarg1_, long jarg2); public final static native long RangeQueryResult_nres_get(long jarg1, RangeQueryResult jarg1_); public final static native void RangeQueryResult_pres_set(long jarg1, RangeQueryResult jarg1_, long jarg2, RangeSearchPartialResult jarg2_); public final static native long RangeQueryResult_pres_get(long jarg1, RangeQueryResult jarg1_); public final static native void RangeQueryResult_add(long jarg1, RangeQueryResult jarg1_, float jarg2, long jarg3); public final static native long new_RangeQueryResult(); public final static native void delete_RangeQueryResult(long jarg1); public final static native void RangeSearchPartialResult_res_set(long jarg1, RangeSearchPartialResult jarg1_, long jarg2, RangeSearchResult jarg2_); public final static native long RangeSearchPartialResult_res_get(long jarg1, RangeSearchPartialResult jarg1_); public final static native void RangeSearchPartialResult_queries_set(long jarg1, RangeSearchPartialResult jarg1_, long jarg2); public final static native long RangeSearchPartialResult_queries_get(long jarg1, RangeSearchPartialResult jarg1_); public final static native long RangeSearchPartialResult_new_result(long jarg1, RangeSearchPartialResult jarg1_, long jarg2); public final static native void RangeSearchPartialResult_set_lims(long jarg1, RangeSearchPartialResult jarg1_); public final static native void RangeSearchPartialResult_copy_result__SWIG_0(long jarg1, RangeSearchPartialResult jarg1_, boolean jarg2); public final static native void RangeSearchPartialResult_copy_result__SWIG_1(long jarg1, RangeSearchPartialResult jarg1_); public final static native void RangeSearchPartialResult_merge__SWIG_0(long jarg1, boolean jarg2); public final static native void RangeSearchPartialResult_merge__SWIG_1(long jarg1); public final static native void delete_RangeSearchPartialResult(long jarg1); public final static native void DistanceComputer_set_query(long jarg1, DistanceComputer jarg1_, long jarg2); public final static native float DistanceComputer_symmetric_dis(long jarg1, DistanceComputer jarg1_, long jarg2, long jarg3); public final static native void delete_DistanceComputer(long jarg1); public final static native boolean InterruptCallback_want_interrupt(long jarg1, InterruptCallback jarg1_); public final static native void delete_InterruptCallback(long jarg1); public final static native void InterruptCallback_clear_instance(); public final static native void InterruptCallback_check(); public final static native boolean InterruptCallback_is_interrupted(); public final static native long InterruptCallback_get_period_hint(long jarg1); public final static native void VisitedTable_visited_set(long jarg1, VisitedTable jarg1_, long jarg2, ByteVector jarg2_); public final static native long VisitedTable_visited_get(long jarg1, VisitedTable jarg1_); public final static native void VisitedTable_visno_set(long jarg1, VisitedTable jarg1_, int jarg2); public final static native int VisitedTable_visno_get(long jarg1, VisitedTable jarg1_); public final static native long new_VisitedTable(int jarg1); public final static native void VisitedTable_set(long jarg1, VisitedTable jarg1_, int jarg2); public final static native boolean VisitedTable_get(long jarg1, VisitedTable jarg1_, int jarg2); public final static native void VisitedTable_advance(long jarg1, VisitedTable jarg1_); public final static native void delete_VisitedTable(long jarg1); public final static native void ignore_SIGTTIN(); public final static native void MapLong2Long_map_set(long jarg1, MapLong2Long jarg1_, long jarg2); public final static native long MapLong2Long_map_get(long jarg1, MapLong2Long jarg1_); public final static native void MapLong2Long_add(long jarg1, MapLong2Long jarg1_, long jarg2, long jarg3, long jarg4); public final static native int MapLong2Long_search(long jarg1, MapLong2Long jarg1_, int jarg2); public final static native void MapLong2Long_search_multiple(long jarg1, MapLong2Long jarg1_, long jarg2, long jarg3, long jarg4); public final static native long new_MapLong2Long(); public final static native void delete_MapLong2Long(long jarg1); public final static native long Clustering_SWIGUpcast(long jarg1); public final static native long Clustering1D_SWIGUpcast(long jarg1); public final static native long ProgressiveDimClusteringParameters_SWIGUpcast(long jarg1); public final static native long ProgressiveDimClustering_SWIGUpcast(long jarg1); public final static native long LinearTransform_SWIGUpcast(long jarg1); public final static native long RandomRotationMatrix_SWIGUpcast(long jarg1); public final static native long PCAMatrix_SWIGUpcast(long jarg1); public final static native long ITQMatrix_SWIGUpcast(long jarg1); public final static native long ITQTransform_SWIGUpcast(long jarg1); public final static native long OPQMatrix_SWIGUpcast(long jarg1); public final static native long RemapDimensionsTransform_SWIGUpcast(long jarg1); public final static native long NormalizationTransform_SWIGUpcast(long jarg1); public final static native long CenteringTransform_SWIGUpcast(long jarg1); public final static native long IndexFlatCodes_SWIGUpcast(long jarg1); public final static native long IndexFlat_SWIGUpcast(long jarg1); public final static native long IndexFlatIP_SWIGUpcast(long jarg1); public final static native long IndexFlatL2_SWIGUpcast(long jarg1); public final static native long IndexFlat1D_SWIGUpcast(long jarg1); public final static native long IndexLSH_SWIGUpcast(long jarg1); public final static native long ReproduceDistancesObjective_SWIGUpcast(long jarg1); public final static native long SimulatedAnnealingOptimizer_SWIGUpcast(long jarg1); public final static native long PolysemousTraining_SWIGUpcast(long jarg1); public final static native long IndexPQ_SWIGUpcast(long jarg1); public final static native long MultiIndexQuantizer_SWIGUpcast(long jarg1); public final static native long MultiIndexQuantizer2_SWIGUpcast(long jarg1); public final static native long ArrayInvertedLists_SWIGUpcast(long jarg1); public final static native long ReadOnlyInvertedLists_SWIGUpcast(long jarg1); public final static native long HStackInvertedLists_SWIGUpcast(long jarg1); public final static native long SliceInvertedLists_SWIGUpcast(long jarg1); public final static native long VStackInvertedLists_SWIGUpcast(long jarg1); public final static native long MaskedInvertedLists_SWIGUpcast(long jarg1); public final static native long StopWordsInvertedLists_SWIGUpcast(long jarg1); public final static native long IndexIVF_SWIGUpcast(long jarg1); public final static native long IndexScalarQuantizer_SWIGUpcast(long jarg1); public final static native long IndexIVFScalarQuantizer_SWIGUpcast(long jarg1); public final static native long IndexHNSW_SWIGUpcast(long jarg1); public final static native long IndexHNSWFlat_SWIGUpcast(long jarg1); public final static native long IndexHNSWPQ_SWIGUpcast(long jarg1); public final static native long IndexHNSWSQ_SWIGUpcast(long jarg1); public final static native long IndexHNSW2Level_SWIGUpcast(long jarg1); public final static native long IndexIVFFlat_SWIGUpcast(long jarg1); public final static native long IndexIVFFlatDedup_SWIGUpcast(long jarg1); public final static native long OnDiskInvertedLists_SWIGUpcast(long jarg1); public final static native long IVFPQSearchParameters_SWIGUpcast(long jarg1); public final static native long IndexIVFPQ_SWIGUpcast(long jarg1); public final static native long Index2Layer_SWIGUpcast(long jarg1); public final static native long IndexBinaryFlat_SWIGUpcast(long jarg1); public final static native long IndexBinaryIVF_SWIGUpcast(long jarg1); public final static native long IndexBinaryFromFloat_SWIGUpcast(long jarg1); public final static native long IndexBinaryHNSW_SWIGUpcast(long jarg1); public final static native long IndexRefine_SWIGUpcast(long jarg1); public final static native long IndexRefineFlat_SWIGUpcast(long jarg1); public final static native long IndexSplitVectors_SWIGUpcast(long jarg1); public final static native long IndexIDMap_SWIGUpcast(long jarg1); public final static native long OneRecallAtRCriterion_SWIGUpcast(long jarg1); public final static native long IntersectionCriterion_SWIGUpcast(long jarg1); public final static native long IDSelectorRange_SWIGUpcast(long jarg1); public final static native long IDSelectorArray_SWIGUpcast(long jarg1); public final static native long IDSelectorBatch_SWIGUpcast(long jarg1); public final static native long RangeSearchPartialResult_SWIGUpcast(long jarg1); } ================================================ FILE: ann/src/main/java/com/twitter/ann/hnsw/BUILD ================================================ java_library( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/guava", "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/twitter/bijection:core", "3rdparty/jvm/commons-lang", "3rdparty/jvm/org/apache/thrift", "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/thrift/com/twitter/ann/common:ann-common-java", "mediaservices/commons/src/main/scala:futuretracker", "scrooge/scrooge-core", "src/java/com/twitter/search/common/file", ], ) ================================================ FILE: ann/src/main/java/com/twitter/ann/hnsw/DistanceFunction.java ================================================ package com.twitter.ann.hnsw; public interface DistanceFunction { /** * Distance between two items. */ float distance(T t, Q q); } ================================================ FILE: ann/src/main/java/com/twitter/ann/hnsw/DistancedItem.java ================================================ package com.twitter.ann.hnsw; /** * An item associated with a float distance * @param The type of the item. */ public class DistancedItem { private final T item; private final float distance; public DistancedItem(T item, float distance) { this.item = item; this.distance = distance; } public T getItem() { return item; } public float getDistance() { return distance; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/hnsw/DistancedItemQueue.java ================================================ package com.twitter.ann.hnsw; import java.util.ArrayList; import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.PriorityQueue; /** * Container for items with their distance. * * @param Type of origin/reference element. * @param Type of element that the queue will hold */ public class DistancedItemQueue implements Iterable> { private final U origin; private final DistanceFunction distFn; private final PriorityQueue> queue; private final boolean minQueue; /** * Creates ontainer for items with their distances. * * @param origin Origin (reference) point * @param initial Initial list of elements to add in the structure * @param minQueue True for min queue, False for max queue * @param distFn Distance function */ public DistancedItemQueue( U origin, List initial, boolean minQueue, DistanceFunction distFn ) { this.origin = origin; this.distFn = distFn; this.minQueue = minQueue; final Comparator> cmp; if (minQueue) { cmp = (o1, o2) -> Float.compare(o1.getDistance(), o2.getDistance()); } else { cmp = (o1, o2) -> Float.compare(o2.getDistance(), o1.getDistance()); } this.queue = new PriorityQueue<>(cmp); enqueueAll(initial); new DistancedItemQueue<>(origin, distFn, queue, minQueue); } private DistancedItemQueue( U origin, DistanceFunction distFn, PriorityQueue> queue, boolean minQueue ) { this.origin = origin; this.distFn = distFn; this.queue = queue; this.minQueue = minQueue; } /** * Enqueues all the items into the queue. */ public void enqueueAll(List list) { for (T t : list) { enqueue(t); } } /** * Return if queue is non empty or not * * @return true if queue is not empty else false */ public boolean nonEmpty() { return !queue.isEmpty(); } /** * Return root of the queue * * @return root of the queue i.e min/max element depending upon min-max queue */ public DistancedItem peek() { return queue.peek(); } /** * Dequeue root of the queue. * * @return remove and return root of the queue i.e min/max element depending upon min-max queue */ public DistancedItem dequeue() { return queue.poll(); } /** * Dequeue all the elements from queueu with ordering mantained * * @return remove all the elements in the order of the queue i.e min/max queue. */ public List> dequeueAll() { final List> list = new ArrayList<>(queue.size()); while (!queue.isEmpty()) { list.add(queue.poll()); } return list; } /** * Convert queue to list * * @return list of elements of queue with distance and without any specific ordering */ public List> toList() { return new ArrayList<>(queue); } /** * Convert queue to list * * @return list of elements of queue without any specific ordering */ List toListWithItem() { List list = new ArrayList<>(queue.size()); Iterator> itr = iterator(); while (itr.hasNext()) { list.add(itr.next().getItem()); } return list; } /** * Enqueue an item into the queue */ public void enqueue(T item) { queue.add(new DistancedItem<>(item, distFn.distance(origin, item))); } /** * Enqueue an item into the queue with its distance. */ public void enqueue(T item, float distance) { queue.add(new DistancedItem<>(item, distance)); } /** * Size * * @return size of the queue */ public int size() { return queue.size(); } /** * Is Min queue * * @return true if min queue else false */ public boolean isMinQueue() { return minQueue; } /** * Returns origin (base element) of the queue * * @return origin of the queue */ public U getOrigin() { return origin; } /** * Return a new queue with ordering reversed. */ public DistancedItemQueue reverse() { final PriorityQueue> rqueue = new PriorityQueue<>(queue.comparator().reversed()); if (queue.isEmpty()) { return new DistancedItemQueue<>(origin, distFn, rqueue, !isMinQueue()); } final Iterator> itr = iterator(); while (itr.hasNext()) { rqueue.add(itr.next()); } return new DistancedItemQueue<>(origin, distFn, rqueue, !isMinQueue()); } @Override public Iterator> iterator() { return queue.iterator(); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/hnsw/HnswIndex.java ================================================ package com.twitter.ann.hnsw; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import org.apache.thrift.TException; import com.twitter.ann.common.IndexOutputFile; import com.twitter.ann.common.thriftjava.HnswInternalIndexMetadata; import com.twitter.bijection.Injection; import com.twitter.logging.Logger; import com.twitter.mediaservices.commons.codec.ArrayByteBufferCodec; import com.twitter.search.common.file.AbstractFile; /** * Typed multithreaded HNSW implementation supporting creation/querying of approximate nearest neighbour * Paper: https://arxiv.org/pdf/1603.09320.pdf * Multithreading impl based on NMSLIB version : https://github.com/nmslib/hnsw/blob/master/hnswlib/hnswalg.h * * @param The type of items inserted / searched in the HNSW index. * @param The type of KNN query. */ public class HnswIndex { private static final Logger LOG = Logger.get(HnswIndex.class); private static final String METADATA_FILE_NAME = "hnsw_internal_metadata"; private static final String GRAPH_FILE_NAME = "hnsw_internal_graph"; private static final int MAP_SIZE_FACTOR = 5; private final DistanceFunction distFnIndex; private final DistanceFunction distFnQuery; private final int efConstruction; private final int maxM; private final int maxM0; private final double levelMultiplier; private final AtomicReference> graphMeta = new AtomicReference<>(); private final Map, ImmutableList> graph; // To take lock on vertex level private final ConcurrentHashMap locks; // To take lock on whole graph only if vertex addition is on layer above the current maxLevel private final ReentrantLock globalLock; private final Function lockProvider; private final RandomProvider randomProvider; // Probability of reevaluating connections of an element in the neighborhood during an update // Can be used as a knob to adjust update_speed/search_speed tradeoff. private final float updateNeighborProbability; /** * Creates instance of hnsw index. * * @param distFnIndex Any distance metric/non metric that specifies similarity between two items for indexing. * @param distFnQuery Any distance metric/non metric that specifies similarity between item for which nearest neighbours queried for and already indexed item. * @param efConstruction Provide speed vs index quality tradeoff, higher the value better the quality and higher the time to create index. * Valid range of efConstruction can be anywhere between 1 and tens of thousand. Typically, it should be set so that a search of M * neighbors with ef=efConstruction should end in recall>0.95. * @param maxM Maximum connections per layer except 0th level. * Optimal values between 5-48. * Smaller M generally produces better result for lower recalls and/ or lower dimensional data, * while bigger M is better for high recall and/ or high dimensional, data on the expense of more memory/disk usage * @param expectedElements Approximate number of elements to be indexed */ protected HnswIndex( DistanceFunction distFnIndex, DistanceFunction distFnQuery, int efConstruction, int maxM, int expectedElements, RandomProvider randomProvider ) { this(distFnIndex, distFnQuery, efConstruction, maxM, expectedElements, new HnswMeta<>(-1, Optional.empty()), new ConcurrentHashMap<>(MAP_SIZE_FACTOR * expectedElements), randomProvider ); } private HnswIndex( DistanceFunction distFnIndex, DistanceFunction distFnQuery, int efConstruction, int maxM, int expectedElements, HnswMeta graphMeta, Map, ImmutableList> graph, RandomProvider randomProvider ) { this.distFnIndex = distFnIndex; this.distFnQuery = distFnQuery; this.efConstruction = efConstruction; this.maxM = maxM; this.maxM0 = 2 * maxM; this.levelMultiplier = 1.0 / Math.log(1.0 * maxM); this.graphMeta.set(graphMeta); this.graph = graph; this.locks = new ConcurrentHashMap<>(MAP_SIZE_FACTOR * expectedElements); this.globalLock = new ReentrantLock(); this.lockProvider = key -> new ReentrantReadWriteLock(); this.randomProvider = randomProvider; this.updateNeighborProbability = 1.0f; } /** * wireConnectionForAllLayers finds connections for a new element and creates bi-direction links. * The method assumes using a reentrant lock to link list reads. * * @param entryPoint the global entry point * @param item the item for which the connections are found * @param itemLevel the level of the added item (maximum layer in which we wire the connections) * @param maxLayer the level of the entry point */ private void wireConnectionForAllLayers(final T entryPoint, final T item, final int itemLevel, final int maxLayer, final boolean isUpdate) { T curObj = entryPoint; if (itemLevel < maxLayer) { curObj = bestEntryPointUntilLayer(curObj, item, maxLayer, itemLevel, distFnIndex); } for (int level = Math.min(itemLevel, maxLayer); level >= 0; level--) { final DistancedItemQueue candidates = searchLayerForCandidates(item, curObj, efConstruction, level, distFnIndex, isUpdate); curObj = mutuallyConnectNewElement(item, candidates, level, isUpdate); } } /** * Insert the item into HNSW index. */ public void insert(final T item) throws IllegalDuplicateInsertException { final Lock itemLock = locks.computeIfAbsent(item, lockProvider).writeLock(); itemLock.lock(); try { final HnswMeta metadata = graphMeta.get(); // If the graph already have the item, should not re-insert it again // Need to check entry point in case we reinsert first item where is are no graph // but only a entry point if (graph.containsKey(HnswNode.from(0, item)) || (metadata.getEntryPoint().isPresent() && Objects.equals(metadata.getEntryPoint().get(), item))) { throw new IllegalDuplicateInsertException( "Duplicate insertion is not supported: " + item); } final int curLevel = getRandomLevel(); Optional entryPoint = metadata.getEntryPoint(); // The global lock prevents two threads from making changes to the entry point. This lock // should get taken very infrequently. Something like log-base-levelMultiplier(num items) // For a full explanation of locking see this document: http://go/hnsw-locking int maxLevelCopy = metadata.getMaxLevel(); if (curLevel > maxLevelCopy) { globalLock.lock(); // Re initialize the entryPoint and maxLevel in case these are changed by any other thread // No need to check the condition again since, // it is already checked at the end before updating entry point struct // No need to unlock for optimization and keeping as is if condition fails since threads // will not be entering this section a lot. final HnswMeta temp = graphMeta.get(); entryPoint = temp.getEntryPoint(); maxLevelCopy = temp.getMaxLevel(); } if (entryPoint.isPresent()) { wireConnectionForAllLayers(entryPoint.get(), item, curLevel, maxLevelCopy, false); } if (curLevel > maxLevelCopy) { Preconditions.checkState(globalLock.isHeldByCurrentThread(), "Global lock not held before updating entry point"); graphMeta.set(new HnswMeta<>(curLevel, Optional.of(item))); } } finally { if (globalLock.isHeldByCurrentThread()) { globalLock.unlock(); } itemLock.unlock(); } } /** * set connections of an element with synchronization * The only other place that should have the lock for writing is during * the element insertion */ private void setConnectionList(final T item, int layer, List connections) { final Lock candidateLock = locks.computeIfAbsent(item, lockProvider).writeLock(); candidateLock.lock(); try { graph.put( HnswNode.from(layer, item), ImmutableList.copyOf(connections) ); } finally { candidateLock.unlock(); } } /** * Reinsert the item into HNSW index. * This method updates the links of an element assuming * the element's distance function is changed externally (e.g. by updating the features) */ public void reInsert(final T item) { final HnswMeta metadata = graphMeta.get(); Optional entryPoint = metadata.getEntryPoint(); Preconditions.checkState(entryPoint.isPresent(), "Update cannot be performed if entry point is not present"); // This is a check for the single element case if (entryPoint.get().equals(item) && graph.isEmpty()) { return; } Preconditions.checkState(graph.containsKey(HnswNode.from(0, item)), "Graph does not contain the item to be updated at level 0"); int curLevel = 0; int maxLevelCopy = metadata.getMaxLevel(); for (int layer = maxLevelCopy; layer >= 0; layer--) { if (graph.containsKey(HnswNode.from(layer, item))) { curLevel = layer; break; } } // Updating the links of the elements from the 1-hop radius of the updated element for (int layer = 0; layer <= curLevel; layer++) { // Filling the element sets for candidates and updated elements final HashSet setCand = new HashSet(); final HashSet setNeigh = new HashSet(); final List listOneHop = getConnectionListForRead(item, layer); if (listOneHop.isEmpty()) { LOG.debug("No links for the updated element. Empty dataset?"); continue; } setCand.add(item); for (T elOneHop : listOneHop) { setCand.add(elOneHop); if (randomProvider.get().nextFloat() > updateNeighborProbability) { continue; } setNeigh.add(elOneHop); final List listTwoHop = getConnectionListForRead(elOneHop, layer); if (listTwoHop.isEmpty()) { LOG.debug("No links for the updated element. Empty dataset?"); } for (T oneHopEl : listTwoHop) { setCand.add(oneHopEl); } } // No need to update the item itself, so remove it setNeigh.remove(item); // Updating the link lists of elements from setNeigh: for (T neigh : setNeigh) { final HashSet setCopy = new HashSet(setCand); setCopy.remove(neigh); int keepElementsNum = Math.min(efConstruction, setCopy.size()); final DistancedItemQueue candidates = new DistancedItemQueue<>( neigh, ImmutableList.of(), false, distFnIndex ); for (T cand : setCopy) { final float distance = distFnIndex.distance(neigh, cand); if (candidates.size() < keepElementsNum) { candidates.enqueue(cand, distance); } else { if (distance < candidates.peek().getDistance()) { candidates.dequeue(); candidates.enqueue(cand, distance); } } } final ImmutableList neighbours = selectNearestNeighboursByHeuristic( candidates, layer == 0 ? maxM0 : maxM ); final List temp = getConnectionListForRead(neigh, layer); if (temp.isEmpty()) { LOG.debug("existing linkslist is empty. Corrupt index"); } if (neighbours.isEmpty()) { LOG.debug("predicted linkslist is empty. Corrupt index"); } setConnectionList(neigh, layer, neighbours); } } wireConnectionForAllLayers(metadata.getEntryPoint().get(), item, curLevel, maxLevelCopy, true); } /** * This method can be used to get the graph statistics, specifically * it prints the histogram of inbound connections for each element. */ private String getStats() { int histogramMaxBins = 50; int[] histogram = new int[histogramMaxBins]; HashMap mmap = new HashMap(); for (HnswNode key : graph.keySet()) { if (key.level == 0) { List linkList = getConnectionListForRead(key.item, key.level); for (T node : linkList) { int a = mmap.computeIfAbsent(node, k -> 0); mmap.put(node, a + 1); } } } for (T key : mmap.keySet()) { int ind = mmap.get(key) < histogramMaxBins - 1 ? mmap.get(key) : histogramMaxBins - 1; histogram[ind]++; } int minNonZeroIndex; for (minNonZeroIndex = histogramMaxBins - 1; minNonZeroIndex >= 0; minNonZeroIndex--) { if (histogram[minNonZeroIndex] > 0) { break; } } String output = ""; for (int i = 0; i <= minNonZeroIndex; i++) { output += "" + i + "\t" + histogram[i] / (0.01f * mmap.keySet().size()) + "\n"; } return output; } private int getRandomLevel() { return (int) (-Math.log(randomProvider.get().nextDouble()) * levelMultiplier); } /** * Note that to avoid deadlocks it is important that this method is called after all the searches * of the graph have completed. If you take a lock on any items discovered in the graph after * this, you may get stuck waiting on a thread that is waiting for item to be fully inserted. *

* Note: when using concurrent writers we can miss connections that we would otherwise get. * This will reduce the recall. *

* For a full explanation of locking see this document: http://go/hnsw-locking * The method returns the closest nearest neighbor (can be used as an enter point) */ private T mutuallyConnectNewElement( final T item, final DistancedItemQueue candidates, // Max queue final int level, final boolean isUpdate ) { // Using maxM here. Its implementation is ambiguous in HNSW paper, // so using the way it is getting used in Hnsw lib. final ImmutableList neighbours = selectNearestNeighboursByHeuristic(candidates, maxM); setConnectionList(item, level, neighbours); final int M = level == 0 ? maxM0 : maxM; for (T nn : neighbours) { if (nn.equals(item)) { continue; } final Lock curLock = locks.computeIfAbsent(nn, lockProvider).writeLock(); curLock.lock(); try { final HnswNode key = HnswNode.from(level, nn); final ImmutableList connections = graph.getOrDefault(key, ImmutableList.of()); final boolean isItemAlreadyPresent = isUpdate && connections.indexOf(item) != -1 ? true : false; // If `item` is already present in the neighboring connections, // then no need to modify any connections or run the search heuristics. if (isItemAlreadyPresent) { continue; } final ImmutableList updatedConnections; if (connections.size() < M) { final List temp = new ArrayList<>(connections); temp.add(item); updatedConnections = ImmutableList.copyOf(temp.iterator()); } else { // Max Queue final DistancedItemQueue queue = new DistancedItemQueue<>( nn, connections, false, distFnIndex ); queue.enqueue(item); updatedConnections = selectNearestNeighboursByHeuristic(queue, M); } if (updatedConnections.isEmpty()) { LOG.debug("Internal error: predicted linkslist is empty"); } graph.put(key, updatedConnections); } finally { curLock.unlock(); } } return neighbours.get(0); } /* * bestEntryPointUntilLayer starts the graph search for item from the entry point * until the searches reaches the selectedLayer layer. * @return a point from selectedLayer layer, was the closest on the (selectedLayer+1) layer */ private T bestEntryPointUntilLayer( final T entryPoint, final K item, int maxLayer, int selectedLayer, DistanceFunction distFn ) { T curObj = entryPoint; if (selectedLayer < maxLayer) { float curDist = distFn.distance(item, curObj); for (int level = maxLayer; level > selectedLayer; level--) { boolean changed = true; while (changed) { changed = false; final List list = getConnectionListForRead(curObj, level); for (T nn : list) { final float tempDist = distFn.distance(item, nn); if (tempDist < curDist) { curDist = tempDist; curObj = nn; changed = true; } } } } } return curObj; } @VisibleForTesting protected ImmutableList selectNearestNeighboursByHeuristic( final DistancedItemQueue candidates, // Max queue final int maxConnections ) { Preconditions.checkState(!candidates.isMinQueue(), "candidates in selectNearestNeighboursByHeuristic should be a max queue"); final T baseElement = candidates.getOrigin(); if (candidates.size() <= maxConnections) { List list = candidates.toListWithItem(); list.remove(baseElement); return ImmutableList.copyOf(list); } else { final List resSet = new ArrayList<>(maxConnections); // Min queue for closest elements first final DistancedItemQueue minQueue = candidates.reverse(); while (minQueue.nonEmpty()) { if (resSet.size() >= maxConnections) { break; } final DistancedItem candidate = minQueue.dequeue(); // We do not want to creates loops: // While heuristic is used only for creating the links if (candidate.getItem().equals(baseElement)) { continue; } boolean toInclude = true; for (T e : resSet) { // Do not include candidate if the distance from candidate to any of existing item in // resSet is closer to the distance from the candidate to the item. By doing this, the // connection of graph will be more diverse, and in case of highly clustered data set, // connections will be made between clusters instead of all being in the same cluster. final float dist = distFnIndex.distance(e, candidate.getItem()); if (dist < candidate.getDistance()) { toInclude = false; break; } } if (toInclude) { resSet.add(candidate.getItem()); } } return ImmutableList.copyOf(resSet); } } /** * Search the index for the neighbours. * * @param query Query * @param numOfNeighbours Number of neighbours to search for. * @param ef This param controls the accuracy of the search. * Bigger the ef better the accuracy on the expense of latency. * Keep it atleast number of neighbours to find. * @return Neighbours */ public List> searchKnn(final Q query, final int numOfNeighbours, final int ef) { final HnswMeta metadata = graphMeta.get(); if (metadata.getEntryPoint().isPresent()) { T entryPoint = bestEntryPointUntilLayer(metadata.getEntryPoint().get(), query, metadata.getMaxLevel(), 0, distFnQuery); // Get the actual neighbours from 0th layer final List> neighbours = searchLayerForCandidates(query, entryPoint, Math.max(ef, numOfNeighbours), 0, distFnQuery, false).dequeueAll(); Collections.reverse(neighbours); return neighbours.size() > numOfNeighbours ? neighbours.subList(0, numOfNeighbours) : neighbours; } else { return Collections.emptyList(); } } // This method is currently not used // It is needed for debugging purposes only private void checkIntegrity(String message) { final HnswMeta metadata = graphMeta.get(); for (HnswNode node : graph.keySet()) { List linkList = graph.get(node); for (T el : linkList) { if (el.equals(node.item)) { LOG.debug(message); throw new RuntimeException("integrity check failed"); } } } } private DistancedItemQueue searchLayerForCandidates( final K item, final T entryPoint, final int ef, final int level, final DistanceFunction distFn, boolean isUpdate ) { // Min queue final DistancedItemQueue cQueue = new DistancedItemQueue<>( item, Collections.singletonList(entryPoint), true, distFn ); // Max Queue final DistancedItemQueue wQueue = cQueue.reverse(); final Set visited = new HashSet<>(); float lowerBoundDistance = wQueue.peek().getDistance(); visited.add(entryPoint); while (cQueue.nonEmpty()) { final DistancedItem candidate = cQueue.peek(); if (candidate.getDistance() > lowerBoundDistance) { break; } cQueue.dequeue(); final List list = getConnectionListForRead(candidate.getItem(), level); for (T nn : list) { if (!visited.contains(nn)) { visited.add(nn); final float distance = distFn.distance(item, nn); if (wQueue.size() < ef || distance < wQueue.peek().getDistance()) { cQueue.enqueue(nn, distance); if (isUpdate && item.equals(nn)) { continue; } wQueue.enqueue(nn, distance); if (wQueue.size() > ef) { wQueue.dequeue(); } lowerBoundDistance = wQueue.peek().getDistance(); } } } } return wQueue; } /** * Serialize hnsw index */ public void toDirectory(IndexOutputFile indexOutputFile, Injection injection) throws IOException, TException { final int totalGraphEntries = HnswIndexIOUtil.saveHnswGraphEntries( graph, indexOutputFile.createFile(GRAPH_FILE_NAME).getOutputStream(), injection); HnswIndexIOUtil.saveMetadata( graphMeta.get(), efConstruction, maxM, totalGraphEntries, injection, indexOutputFile.createFile(METADATA_FILE_NAME).getOutputStream()); } /** * Load hnsw index */ public static HnswIndex loadHnswIndex( DistanceFunction distFnIndex, DistanceFunction distFnQuery, AbstractFile directory, Injection injection, RandomProvider randomProvider) throws IOException, TException { final AbstractFile graphFile = directory.getChild(GRAPH_FILE_NAME); final AbstractFile metadataFile = directory.getChild(METADATA_FILE_NAME); final HnswInternalIndexMetadata metadata = HnswIndexIOUtil.loadMetadata(metadataFile); final Map, ImmutableList> graph = HnswIndexIOUtil.loadHnswGraph(graphFile, injection, metadata.numElements); final ByteBuffer entryPointBB = metadata.entryPoint; final HnswMeta graphMeta = new HnswMeta<>( metadata.maxLevel, entryPointBB == null ? Optional.empty() : Optional.of(injection.invert(ArrayByteBufferCodec.decode(entryPointBB)).get()) ); return new HnswIndex<>( distFnIndex, distFnQuery, metadata.efConstruction, metadata.maxM, metadata.numElements, graphMeta, graph, randomProvider ); } private List getConnectionListForRead(T node, int level) { final Lock curLock = locks.computeIfAbsent(node, lockProvider).readLock(); curLock.lock(); final List list; try { list = graph .getOrDefault(HnswNode.from(level, node), ImmutableList.of()); } finally { curLock.unlock(); } return list; } @VisibleForTesting AtomicReference> getGraphMeta() { return graphMeta; } @VisibleForTesting Map getLocks() { return locks; } @VisibleForTesting Map, ImmutableList> getGraph() { return graph; } public interface RandomProvider { /** * RandomProvider interface made public for scala 2.12 compat */ Random get(); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/hnsw/HnswIndexIOUtil.java ================================================ package com.twitter.ann.hnsw; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import com.google.common.collect.ImmutableList; import org.apache.thrift.TDeserializer; import org.apache.thrift.TException; import org.apache.thrift.TSerializer; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.transport.TIOStreamTransport; import org.apache.thrift.transport.TTransportException; import com.twitter.ann.common.thriftjava.HnswGraphEntry; import com.twitter.ann.common.thriftjava.HnswInternalIndexMetadata; import com.twitter.bijection.Injection; import com.twitter.mediaservices.commons.codec.ArrayByteBufferCodec; import com.twitter.search.common.file.AbstractFile; public final class HnswIndexIOUtil { private HnswIndexIOUtil() { } /** * Save thrift object in file */ public static void saveMetadata( HnswMeta graphMeta, int efConstruction, int maxM, int numElements, Injection injection, OutputStream outputStream ) throws IOException, TException { final int maxLevel = graphMeta.getMaxLevel(); final HnswInternalIndexMetadata metadata = new HnswInternalIndexMetadata( maxLevel, efConstruction, maxM, numElements ); if (graphMeta.getEntryPoint().isPresent()) { metadata.setEntryPoint(injection.apply(graphMeta.getEntryPoint().get())); } final TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory()); outputStream.write(serializer.serialize(metadata)); outputStream.close(); } /** * Load Hnsw index metadata */ public static HnswInternalIndexMetadata loadMetadata(AbstractFile file) throws IOException, TException { final HnswInternalIndexMetadata obj = new HnswInternalIndexMetadata(); final TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory()); deserializer.deserialize(obj, file.getByteSource().read()); return obj; } /** * Load Hnsw graph entries from file */ public static Map, ImmutableList> loadHnswGraph( AbstractFile file, Injection injection, int numElements ) throws IOException, TException { final InputStream stream = file.getByteSource().openBufferedStream(); final TProtocol protocol = new TBinaryProtocol(new TIOStreamTransport(stream)); final Map, ImmutableList> graph = new HashMap<>(numElements); while (true) { try { final HnswGraphEntry entry = new HnswGraphEntry(); entry.read(protocol); final HnswNode node = HnswNode.from(entry.level, injection.invert(ArrayByteBufferCodec.decode(entry.key)).get()); final List list = entry.getNeighbours().stream() .map(bb -> injection.invert(ArrayByteBufferCodec.decode(bb)).get()) .collect(Collectors.toList()); graph.put(node, ImmutableList.copyOf(list.iterator())); } catch (TException e) { if (e instanceof TTransportException && TTransportException.class.cast(e).getType() == TTransportException.END_OF_FILE) { stream.close(); break; } stream.close(); throw e; } } return graph; } /** * Save hnsw graph in file * * @return number of keys in the graph */ public static int saveHnswGraphEntries( Map, ImmutableList> graph, OutputStream outputStream, Injection injection ) throws IOException, TException { final TProtocol protocol = new TBinaryProtocol(new TIOStreamTransport(outputStream)); final Set> nodes = graph.keySet(); for (HnswNode node : nodes) { final HnswGraphEntry entry = new HnswGraphEntry(); entry.setLevel(node.level); entry.setKey(injection.apply(node.item)); final List nn = graph.getOrDefault(node, ImmutableList.of()).stream() .map(t -> ByteBuffer.wrap(injection.apply(t))) .collect(Collectors.toList()); entry.setNeighbours(nn); entry.write(protocol); } outputStream.close(); return nodes.size(); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/hnsw/HnswMeta.java ================================================ package com.twitter.ann.hnsw; import java.util.Objects; import java.util.Optional; class HnswMeta { private final int maxLevel; private final Optional entryPoint; HnswMeta(int maxLevel, Optional entryPoint) { this.maxLevel = maxLevel; this.entryPoint = entryPoint; } public int getMaxLevel() { return maxLevel; } public Optional getEntryPoint() { return entryPoint; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } HnswMeta hnswMeta = (HnswMeta) o; return maxLevel == hnswMeta.maxLevel && Objects.equals(entryPoint, hnswMeta.entryPoint); } @Override public int hashCode() { return Objects.hash(maxLevel, entryPoint); } @Override public String toString() { return "HnswMeta{maxLevel=" + maxLevel + ", entryPoint=" + entryPoint + '}'; } } ================================================ FILE: ann/src/main/java/com/twitter/ann/hnsw/HnswNode.java ================================================ package com.twitter.ann.hnsw; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; public class HnswNode { public final int level; public final T item; public HnswNode(int level, T item) { this.level = level; this.item = item; } /** * Create a hnsw node. */ public static HnswNode from(int level, T item) { return new HnswNode<>(level, item); } @Override public boolean equals(Object o) { if (o == this) { return true; } if (!(o instanceof HnswNode)) { return false; } HnswNode that = (HnswNode) o; return new EqualsBuilder() .append(this.item, that.item) .append(this.level, that.level) .isEquals(); } @Override public int hashCode() { return new HashCodeBuilder() .append(item) .append(level) .toHashCode(); } } ================================================ FILE: ann/src/main/java/com/twitter/ann/hnsw/IllegalDuplicateInsertException.java ================================================ package com.twitter.ann.hnsw; public class IllegalDuplicateInsertException extends Exception { public IllegalDuplicateInsertException(String message) { super(message); } } ================================================ FILE: ann/src/main/python/dataflow/BUILD.bazel ================================================ resources( name = "sql", sources = ["bq.sql"], ) python3_library( name = "faiss_indexing", sources = ["**/*.py"], tags = ["bazel-compatible"], dependencies = [ ":sql", "3rdparty/python/apache-beam:default", "3rdparty/python/faiss-gpu:default", "3rdparty/python/gcsfs:default", "3rdparty/python/google-cloud-bigquery:default", "3rdparty/python/google-cloud-storage", "3rdparty/python/numpy:default", "3rdparty/python/pandas:default", "3rdparty/python/pandas-gbq:default", "3rdparty/python/pyarrow:default", "src/python/twitter/ml/common/apache_beam", ], ) python37_binary( name = "faiss_indexing_bin", sources = ["faiss_index_bq_dataset.py"], platforms = [ "current", "linux_x86_64", ], tags = ["no-mypy"], zip_safe = False, dependencies = [ ":faiss_indexing", "3rdparty/python/_closures/ann/src/main/python/dataflow:faiss_indexing_bin", ], ) ================================================ FILE: ann/src/main/python/dataflow/bq.sql ================================================ WITH maxts as (SELECT as value MAX(ts) as ts FROM `twttr-recos-ml-prod.ssedhain.twhin_tweet_avg_embedding`) SELECT entityId, embedding FROM `twttr-recos-ml-prod.ssedhain.twhin_tweet_avg_embedding` WHERE ts >= (select max(maxts) from maxts) AND DATE(TIMESTAMP_MILLIS(createdAt)) <= (select max(maxts) from maxts) AND DATE(TIMESTAMP_MILLIS(createdAt)) >= DATE_SUB((select max(maxts) from maxts), INTERVAL 1 DAY) ================================================ FILE: ann/src/main/python/dataflow/faiss_index_bq_dataset.py ================================================ import argparse import logging import os import pkgutil import sys from urllib.parse import urlsplit import apache_beam as beam from apache_beam.options.pipeline_options import PipelineOptions import faiss def parse_d6w_config(argv=None): """Parse d6w config. :param argv: d6w config :return: dictionary containing d6w config """ parser = argparse.ArgumentParser( description="See https://docbird.twitter.biz/d6w/model.html for any parameters inherited from d6w job config" ) parser.add_argument("--job_name", dest="job_name", required=True, help="d6w attribute") parser.add_argument("--project", dest="project", required=True, help="d6w attribute") parser.add_argument( "--staging_location", dest="staging_location", required=True, help="d6w attribute" ) parser.add_argument("--temp_location", dest="temp_location", required=True, help="d6w attribute") parser.add_argument( "--output_location", dest="output_location", required=True, help="GCS bucket and path where resulting artifacts are uploaded", ) parser.add_argument( "--service_account_email", dest="service_account_email", required=True, help="d6w attribute" ) parser.add_argument( "--factory_string", dest="factory_string", required=False, help="FAISS factory string describing index to build. See https://github.com/facebookresearch/faiss/wiki/The-index-factory", ) parser.add_argument( "--metric", dest="metric", required=True, help="Metric used to compute distance between embeddings. Valid values are 'l2', 'ip', 'l1', 'linf'", ) parser.add_argument( "--use_gpu", dest="gpu", required=True, help="--use_gpu=yes if you want to use GPU during index building", ) known_args, unknown_args = parser.parse_known_args(argv) d6w_config = vars(known_args) d6w_config["gpu"] = d6w_config["gpu"].lower() == "yes" d6w_config["metric"] = parse_metric(d6w_config) """ WARNING: Currently, d6w (a Twitter tool used to deploy Dataflow jobs to GCP) and PipelineOptions.for_dataflow_runner (a helper method in twitter.ml.common.apache_beam) do not play nicely together. The helper method will overwrite some of the config specified in the d6w file using the defaults in https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/src/python/twitter/ml/common/apache_beam/__init__.py?L24.' However, the d6w output message will still report that the config specified in the d6w file was used. """ logging.warning( f"The following d6w config parameters will be overwritten by the defaults in " f"https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/src/python/twitter/ml/common/apache_beam/__init__.py?L24\n" f"{str(unknown_args)}" ) return d6w_config def get_bq_query(): """ Query is expected to return rows with unique entityId """ return pkgutil.get_data(__name__, "bq.sql").decode("utf-8") def parse_metric(config): metric_str = config["metric"].lower() if metric_str == "l2": return faiss.METRIC_L2 elif metric_str == "ip": return faiss.METRIC_INNER_PRODUCT elif metric_str == "l1": return faiss.METRIC_L1 elif metric_str == "linf": return faiss.METRIC_Linf else: raise Exception(f"Unknown metric: {metric_str}") def run_pipeline(argv=[]): config = parse_d6w_config(argv) argv_with_extras = argv if config["gpu"]: argv_with_extras.extend(["--experiments", "use_runner_v2"]) argv_with_extras.extend( ["--experiments", "worker_accelerator=type:nvidia-tesla-t4;count:1;install-nvidia-driver"] ) argv_with_extras.extend( [ "--worker_harness_container_image", "gcr.io/twttr-recos-ml-prod/dataflow-gpu/beam2_39_0_py3_7", ] ) options = PipelineOptions(argv_with_extras) output_bucket_name = urlsplit(config["output_location"]).netloc with beam.Pipeline(options=options) as p: input_data = p | "Read from BigQuery" >> beam.io.ReadFromBigQuery( method=beam.io.ReadFromBigQuery.Method.DIRECT_READ, query=get_bq_query(), use_standard_sql=True, ) index_built = input_data | "Build and upload index" >> beam.CombineGlobally( MergeAndBuildIndex( output_bucket_name, config["output_location"], config["factory_string"], config["metric"], config["gpu"], ) ) # Make linter happy index_built class MergeAndBuildIndex(beam.CombineFn): def __init__(self, bucket_name, gcs_output_path, factory_string, metric, gpu): self.bucket_name = bucket_name self.gcs_output_path = gcs_output_path self.factory_string = factory_string self.metric = metric self.gpu = gpu def create_accumulator(self): return [] def add_input(self, accumulator, element): accumulator.append(element) return accumulator def merge_accumulators(self, accumulators): merged = [] for accum in accumulators: merged.extend(accum) return merged def extract_output(self, rows): # Reimports are needed on workers import glob import subprocess import faiss from google.cloud import storage import numpy as np client = storage.Client() bucket = client.get_bucket(self.bucket_name) logging.info("Building FAISS index") logging.info(f"There are {len(rows)} rows") ids = np.array([x["entityId"] for x in rows]).astype("long") embeds = np.array([x["embedding"] for x in rows]).astype("float32") dimensions = len(embeds[0]) N = ids.shape[0] logging.info(f"There are {dimensions} dimensions") if self.factory_string is None: M = 48 divideable_dimensions = (dimensions // M) * M if divideable_dimensions != dimensions: opq_prefix = f"OPQ{M}_{divideable_dimensions}" else: opq_prefix = f"OPQ{M}" clusters = N // 20 self.factory_string = f"{opq_prefix},IVF{clusters},PQ{M}" logging.info(f"Factory string is {self.factory_string}, metric={self.metric}") if self.gpu: logging.info("Using GPU") res = faiss.StandardGpuResources() cpu_index = faiss.index_factory(dimensions, self.factory_string, self.metric) cpu_index = faiss.IndexIDMap(cpu_index) gpu_index = faiss.index_cpu_to_gpu(res, 0, cpu_index) gpu_index.train(embeds) gpu_index.add_with_ids(embeds, ids) cpu_index = faiss.index_gpu_to_cpu(gpu_index) else: logging.info("Using CPU") cpu_index = faiss.index_factory(dimensions, self.factory_string, self.metric) cpu_index = faiss.IndexIDMap(cpu_index) cpu_index.train(embeds) cpu_index.add_with_ids(embeds, ids) logging.info("Built faiss index") local_path = "/indices" logging.info(f"Writing indices to local {local_path}") subprocess.run(f"mkdir -p {local_path}".strip().split()) local_index_path = os.path.join(local_path, "result.index") faiss.write_index(cpu_index, local_index_path) logging.info(f"Done writing indices to local {local_path}") logging.info(f"Uploading to GCS with path {self.gcs_output_path}") assert os.path.isdir(local_path) for local_file in glob.glob(local_path + "/*"): remote_path = os.path.join( self.gcs_output_path.split("/")[-1], local_file[1 + len(local_path) :] ) blob = bucket.blob(remote_path) blob.upload_from_filename(local_file) if __name__ == "__main__": logging.getLogger().setLevel(logging.INFO) run_pipeline(sys.argv) ================================================ FILE: ann/src/main/python/dataflow/worker_harness/Dockerfile ================================================ FROM --platform=linux/amd64 nvidia/cuda:11.2.2-cudnn8-runtime-ubuntu20.04 RUN \ # Add Deadsnakes repository that has a variety of Python packages for Ubuntu. # See: https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F23C5A6CF475977595C89F51BA6932366A755776 \ && echo "deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main" >> /etc/apt/sources.list.d/custom.list \ && echo "deb-src http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main" >> /etc/apt/sources.list.d/custom.list \ && apt-get update \ && apt-get install -y curl \ python3.7 \ # With python3.8 package, distutils need to be installed separately. python3.7-distutils \ python3-dev \ python3.7-dev \ libpython3.7-dev \ python3-apt \ gcc \ g++ \ && rm -rf /var/lib/apt/lists/* RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 10 RUN rm -f /usr/bin/python3 && ln -s /usr/bin/python3.7 /usr/bin/python3 RUN \ curl https://bootstrap.pypa.io/get-pip.py | python \ && pip3 install pip==22.0.3 \ && python3 -m pip install --no-cache-dir apache-beam[gcp]==2.39.0 # Verify that there are no conflicting dependencies. RUN pip3 check # Copy the Apache Beam worker dependencies from the Beam Python 3.7 SDK image. COPY --from=apache/beam_python3.7_sdk:2.39.0 /opt/apache/beam /opt/apache/beam # Set the entrypoint to Apache Beam SDK worker launcher. ENTRYPOINT [ "/opt/apache/beam/boot" ] ================================================ FILE: ann/src/main/python/dataflow/worker_harness/cloudbuild.yml ================================================ steps: - name: 'gcr.io/cloud-builders/docker' args: ['build', '-t', 'gcr.io/twttr-recos-ml-prod/dataflow-gpu/beam2_39_0_py3_7', '.'] - name: 'gcr.io/cloud-builders/docker' args: ['push', 'gcr.io/twttr-recos-ml-prod/dataflow-gpu/beam2_39_0_py3_7'] images: ['gcr.io/twttr-recos-ml-prod/dataflow-gpu/beam2_39_0_py3_7'] ================================================ FILE: ann/src/main/scala/com/twitter/ann/annoy/AnnoyCommon.scala ================================================ package com.twitter.ann.annoy import com.twitter.ann.common.RuntimeParams import com.twitter.ann.common.thriftscala.AnnoyIndexMetadata import com.twitter.bijection.Injection import com.twitter.mediaservices.commons.codec.ThriftByteBufferCodec import com.twitter.ann.common.thriftscala.{AnnoyRuntimeParam, RuntimeParams => ServiceRuntimeParams} import scala.util.{Failure, Success, Try} object AnnoyCommon { private[annoy] lazy val MetadataCodec = new ThriftByteBufferCodec(AnnoyIndexMetadata) private[annoy] val IndexFileName = "annoy_index" private[annoy] val MetaDataFileName = "annoy_index_metadata" private[annoy] val IndexIdMappingFileName = "annoy_index_id_mapping" val RuntimeParamsInjection: Injection[AnnoyRuntimeParams, ServiceRuntimeParams] = new Injection[AnnoyRuntimeParams, ServiceRuntimeParams] { override def apply(scalaParams: AnnoyRuntimeParams): ServiceRuntimeParams = { ServiceRuntimeParams.AnnoyParam( AnnoyRuntimeParam( scalaParams.nodesToExplore ) ) } override def invert(thriftParams: ServiceRuntimeParams): Try[AnnoyRuntimeParams] = thriftParams match { case ServiceRuntimeParams.AnnoyParam(annoyParam) => Success( AnnoyRuntimeParams(annoyParam.numOfNodesToExplore) ) case p => Failure(new IllegalArgumentException(s"Expected AnnoyRuntimeParams got $p")) } } } case class AnnoyRuntimeParams( /* Number of vectors to evaluate while searching. A larger value will give more accurate results, but will take longer time to return. * Default value would be numberOfTrees*numberOfNeigboursRequested */ nodesToExplore: Option[Int]) extends RuntimeParams { override def toString: String = s"AnnoyRuntimeParams( nodesToExplore = $nodesToExplore)" } ================================================ FILE: ann/src/main/scala/com/twitter/ann/annoy/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/spotify:annoy-java", "3rdparty/jvm/com/spotify:annoy-snapshot", "3rdparty/jvm/com/twitter/storehaus:core", "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/scala/com/twitter/ann/file_store", "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", "mediaservices/commons", "src/java/com/twitter/search/common/file", "src/scala/com/twitter/ml/api/embedding", ], exports = [ "ann/src/main/scala/com/twitter/ann/common", "src/java/com/twitter/common_internal/hadoop", "src/java/com/twitter/search/common/file", "src/scala/com/twitter/ml/api/embedding", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/annoy/RawAnnoyIndexBuilder.scala ================================================ package com.twitter.ann.annoy import com.spotify.annoy.jni.base.{Annoy => AnnoyLib} import com.twitter.ann.annoy.AnnoyCommon.IndexFileName import com.twitter.ann.annoy.AnnoyCommon.MetaDataFileName import com.twitter.ann.annoy.AnnoyCommon.MetadataCodec import com.twitter.ann.common.EmbeddingType._ import com.twitter.ann.common._ import com.twitter.ann.common.thriftscala.AnnoyIndexMetadata import com.twitter.concurrent.AsyncSemaphore import com.twitter.mediaservices.commons.codec.ArrayByteBufferCodec import com.twitter.search.common.file.AbstractFile import com.twitter.search.common.file.LocalFile import com.twitter.util.Future import com.twitter.util.FuturePool import java.io.File import java.nio.file.Files import org.apache.beam.sdk.io.fs.ResourceId import scala.collection.JavaConverters._ private[annoy] object RawAnnoyIndexBuilder { private[annoy] def apply[D <: Distance[D]]( dimension: Int, numOfTrees: Int, metric: Metric[D], futurePool: FuturePool ): RawAppendable[AnnoyRuntimeParams, D] with Serialization = { val indexBuilder = AnnoyLib.newIndex(dimension, annoyMetric(metric)) new RawAnnoyIndexBuilder(dimension, numOfTrees, metric, indexBuilder, futurePool) } private[this] def annoyMetric(metric: Metric[_]): AnnoyLib.Metric = { metric match { case L2 => AnnoyLib.Metric.EUCLIDEAN case Cosine => AnnoyLib.Metric.ANGULAR case _ => throw new RuntimeException("Not supported: " + metric) } } } private[this] class RawAnnoyIndexBuilder[D <: Distance[D]]( dimension: Int, numOfTrees: Int, metric: Metric[D], indexBuilder: AnnoyLib.Builder, futurePool: FuturePool) extends RawAppendable[AnnoyRuntimeParams, D] with Serialization { private[this] var counter = 0 // Note: Only one thread can access the underlying index, multithreaded index building not supported private[this] val semaphore = new AsyncSemaphore(1) override def append(embedding: EmbeddingVector): Future[Long] = semaphore.acquireAndRun({ counter += 1 indexBuilder.addItem( counter, embedding.toArray .map(float => float2Float(float)) .toList .asJava ) Future.value(counter) }) override def toQueryable: Queryable[Long, AnnoyRuntimeParams, D] = { val tempDirParent = Files.createTempDirectory("raw_annoy_index").toFile tempDirParent.deleteOnExit val tempDir = new LocalFile(tempDirParent) this.toDirectory(tempDir) RawAnnoyQueryIndex( dimension, metric, futurePool, tempDir ) } override def toDirectory(directory: ResourceId): Unit = { toDirectory(new IndexOutputFile(directory)) } /** * Serialize the annoy index in a directory. * @param directory: Directory to save to. */ override def toDirectory(directory: AbstractFile): Unit = { toDirectory(new IndexOutputFile(directory)) } private def toDirectory(directory: IndexOutputFile): Unit = { val indexFile = directory.createFile(IndexFileName) saveIndex(indexFile) val metaDataFile = directory.createFile(MetaDataFileName) saveMetadata(metaDataFile) } private[this] def saveIndex(indexFile: IndexOutputFile): Unit = { val index = indexBuilder .build(numOfTrees) val temp = new LocalFile(File.createTempFile(IndexFileName, null)) index.save(temp.getPath) indexFile.copyFrom(temp.getByteSource.openStream()) temp.delete() } private[this] def saveMetadata(metadataFile: IndexOutputFile): Unit = { val numberOfVectorsIndexed = counter val metadata = AnnoyIndexMetadata( dimension, Metric.toThrift(metric), numOfTrees, numberOfVectorsIndexed ) val bytes = ArrayByteBufferCodec.decode(MetadataCodec.encode(metadata)) val temp = new LocalFile(File.createTempFile(MetaDataFileName, null)) temp.getByteSink.write(bytes) metadataFile.copyFrom(temp.getByteSource.openStream()) temp.delete() } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/annoy/RawAnnoyQueryIndex.scala ================================================ package com.twitter.ann.annoy import com.spotify.annoy.{ANNIndex, IndexType} import com.twitter.ann.annoy.AnnoyCommon._ import com.twitter.ann.common._ import com.twitter.ann.common.EmbeddingType._ import com.twitter.mediaservices.commons.codec.ArrayByteBufferCodec import com.twitter.search.common.file.{AbstractFile, LocalFile} import com.twitter.util.{Future, FuturePool} import java.io.File import scala.collection.JavaConverters._ private[annoy] object RawAnnoyQueryIndex { private[annoy] def apply[D <: Distance[D]]( dimension: Int, metric: Metric[D], futurePool: FuturePool, directory: AbstractFile ): Queryable[Long, AnnoyRuntimeParams, D] = { val metadataFile = directory.getChild(MetaDataFileName) val indexFile = directory.getChild(IndexFileName) val metadata = MetadataCodec.decode( ArrayByteBufferCodec.encode(metadataFile.getByteSource.read()) ) val existingDimension = metadata.dimension assert( existingDimension == dimension, s"Dimensions do not match. requested: $dimension existing: $existingDimension" ) val existingMetric = Metric.fromThrift(metadata.distanceMetric) assert( existingMetric == metric, s"DistanceMetric do not match. requested: $metric existing: $existingMetric" ) val index = loadIndex(indexFile, dimension, annoyMetric(metric)) new RawAnnoyQueryIndex[D]( dimension, metric, metadata.numOfTrees, index, futurePool ) } private[this] def annoyMetric(metric: Metric[_]): IndexType = { metric match { case L2 => IndexType.EUCLIDEAN case Cosine => IndexType.ANGULAR case _ => throw new RuntimeException("Not supported: " + metric) } } private[this] def loadIndex( indexFile: AbstractFile, dimension: Int, indexType: IndexType ): ANNIndex = { var localIndexFile = indexFile // If not a local file copy to local, so that it can be memory mapped. if (!indexFile.isInstanceOf[LocalFile]) { val tempFile = File.createTempFile(IndexFileName, null) tempFile.deleteOnExit() val temp = new LocalFile(tempFile) indexFile.copyTo(temp) localIndexFile = temp } new ANNIndex( dimension, localIndexFile.getPath(), indexType ) } } private[this] class RawAnnoyQueryIndex[D <: Distance[D]]( dimension: Int, metric: Metric[D], numOfTrees: Int, index: ANNIndex, futurePool: FuturePool) extends Queryable[Long, AnnoyRuntimeParams, D] with AutoCloseable { override def query( embedding: EmbeddingVector, numOfNeighbours: Int, runtimeParams: AnnoyRuntimeParams ): Future[List[Long]] = { queryWithDistance(embedding, numOfNeighbours, runtimeParams) .map(_.map(_.neighbor)) } override def queryWithDistance( embedding: EmbeddingVector, numOfNeighbours: Int, runtimeParams: AnnoyRuntimeParams ): Future[List[NeighborWithDistance[Long, D]]] = { futurePool { val queryVector = embedding.toArray val neigboursToRequest = neighboursToRequest(numOfNeighbours, runtimeParams) val neigbours = index .getNearestWithDistance(queryVector, neigboursToRequest) .asScala .take(numOfNeighbours) .map { nn => val id = nn.getFirst.toLong val distance = metric.fromAbsoluteDistance(nn.getSecond) NeighborWithDistance(id, distance) } .toList neigbours } } // Annoy java lib do not expose param for numOfNodesToExplore. // Default number is numOfTrees*numOfNeigbours. // Simple hack is to artificially increase the numOfNeighbours to be requested and then just cap it before returning. private[this] def neighboursToRequest( numOfNeighbours: Int, annoyParams: AnnoyRuntimeParams ): Int = { annoyParams.nodesToExplore match { case Some(nodesToExplore) => { val neigboursToRequest = nodesToExplore / numOfTrees if (neigboursToRequest < numOfNeighbours) numOfNeighbours else neigboursToRequest } case _ => numOfNeighbours } } // To close the memory map based file resource. override def close(): Unit = index.close() } ================================================ FILE: ann/src/main/scala/com/twitter/ann/annoy/TypedAnnoyIndex.scala ================================================ package com.twitter.ann.annoy import com.twitter.ann.common._ import com.twitter.bijection.Injection import com.twitter.search.common.file.AbstractFile import com.twitter.util.FuturePool // Class to provide Annoy based ann index. object TypedAnnoyIndex { /** * Create Annoy based typed index builder that serializes index to a directory (HDFS/Local file system). * It cannot be used in scalding as it leverage C/C++ jni bindings, whose build conflicts with version of some libs installed on hadoop. * You can use it on aurora or with IndexBuilding job which triggers scalding job but then streams data to aurora machine for building index. * @param dimension dimension of embedding * @param numOfTrees builds a forest of numOfTrees trees. * More trees gives higher precision when querying at the cost of increased memory and disk storage requirement at the build time. * At runtime the index will be memory mapped, so memory wont be an issue but disk storage would be needed. * @param metric distance metric for nearest neighbour search * @param injection Injection to convert bytes to Id. * @tparam T Type of Id for embedding * @tparam D Typed Distance * @return Serializable AnnoyIndex */ def indexBuilder[T, D <: Distance[D]]( dimension: Int, numOfTrees: Int, metric: Metric[D], injection: Injection[T, Array[Byte]], futurePool: FuturePool ): Appendable[T, AnnoyRuntimeParams, D] with Serialization = { TypedAnnoyIndexBuilderWithFile(dimension, numOfTrees, metric, injection, futurePool) } /** * Load Annoy based queryable index from a directory * @param dimension dimension of embedding * @param metric distance metric for nearest neighbour search * @param injection Injection to convert bytes to Id. * @param futurePool FuturePool * @param directory Directory (HDFS/Local file system) where serialized index is stored. * @tparam T Type of Id for embedding * @tparam D Typed Distance * @return Typed Queryable AnnoyIndex */ def loadQueryableIndex[T, D <: Distance[D]]( dimension: Int, metric: Metric[D], injection: Injection[T, Array[Byte]], futurePool: FuturePool, directory: AbstractFile ): Queryable[T, AnnoyRuntimeParams, D] = { TypedAnnoyQueryIndexWithFile(dimension, metric, injection, futurePool, directory) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/annoy/TypedAnnoyIndexBuilderWithFile.scala ================================================ package com.twitter.ann.annoy import com.twitter.ann.annoy.AnnoyCommon.IndexIdMappingFileName import com.twitter.ann.common._ import com.twitter.ann.file_store.WritableIndexIdFileStore import com.twitter.bijection.Injection import com.twitter.search.common.file.AbstractFile import com.twitter.util.Future import com.twitter.util.FuturePool import org.apache.beam.sdk.io.fs.ResourceId private[annoy] object TypedAnnoyIndexBuilderWithFile { private[annoy] def apply[T, D <: Distance[D]]( dimension: Int, numOfTrees: Int, metric: Metric[D], injection: Injection[T, Array[Byte]], futurePool: FuturePool ): Appendable[T, AnnoyRuntimeParams, D] with Serialization = { val index = RawAnnoyIndexBuilder(dimension, numOfTrees, metric, futurePool) val writableFileStore = WritableIndexIdFileStore(injection) new TypedAnnoyIndexBuilderWithFile[T, D](index, writableFileStore) } } private[this] class TypedAnnoyIndexBuilderWithFile[T, D <: Distance[D]]( indexBuilder: RawAppendable[AnnoyRuntimeParams, D] with Serialization, store: WritableIndexIdFileStore[T]) extends Appendable[T, AnnoyRuntimeParams, D] with Serialization { private[this] val transformedIndex = IndexTransformer.transformAppendable(indexBuilder, store) override def append(entity: EntityEmbedding[T]): Future[Unit] = { transformedIndex.append(entity) } override def toDirectory(directory: ResourceId): Unit = { indexBuilder.toDirectory(directory) toDirectory(new IndexOutputFile(directory)) } override def toDirectory(directory: AbstractFile): Unit = { indexBuilder.toDirectory(directory) toDirectory(new IndexOutputFile(directory)) } private def toDirectory(directory: IndexOutputFile): Unit = { val indexIdFile = directory.createFile(IndexIdMappingFileName) store.save(indexIdFile) } override def toQueryable: Queryable[T, AnnoyRuntimeParams, D] = { transformedIndex.toQueryable } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/annoy/TypedAnnoyQueryIndexWithFile.scala ================================================ package com.twitter.ann.annoy import com.twitter.ann.annoy.AnnoyCommon._ import com.twitter.ann.common._ import com.twitter.ann.file_store.ReadableIndexIdFileStore import com.twitter.bijection.Injection import com.twitter.search.common.file.AbstractFile import com.twitter.util.FuturePool private[annoy] object TypedAnnoyQueryIndexWithFile { private[annoy] def apply[T, D <: Distance[D]]( dimension: Int, metric: Metric[D], injection: Injection[T, Array[Byte]], futurePool: FuturePool, directory: AbstractFile ): Queryable[T, AnnoyRuntimeParams, D] = { val deserializer = new TypedAnnoyQueryIndexWithFile(dimension, metric, futurePool, injection) deserializer.fromDirectory(directory) } } private[this] class TypedAnnoyQueryIndexWithFile[T, D <: Distance[D]]( dimension: Int, metric: Metric[D], futurePool: FuturePool, injection: Injection[T, Array[Byte]]) extends QueryableDeserialization[ T, AnnoyRuntimeParams, D, Queryable[T, AnnoyRuntimeParams, D] ] { override def fromDirectory(directory: AbstractFile): Queryable[T, AnnoyRuntimeParams, D] = { val index = RawAnnoyQueryIndex(dimension, metric, futurePool, directory) val indexIdFile = directory.getChild(IndexIdMappingFileName) val readableFileStore = ReadableIndexIdFileStore(indexIdFile, injection) IndexTransformer.transformQueryable(index, readableFileStore) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/brute_force/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/scala/com/twitter/ann/serialization", "ann/src/main/thrift/com/twitter/ann/serialization:serialization-scala", "src/java/com/twitter/search/common/file", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/brute_force/BruteForceDeserialization.scala ================================================ package com.twitter.ann.brute_force import com.google.common.annotations.VisibleForTesting import com.twitter.ann.common.{Distance, EntityEmbedding, Metric, QueryableDeserialization} import com.twitter.ann.serialization.{PersistedEmbeddingInjection, ThriftIteratorIO} import com.twitter.ann.serialization.thriftscala.PersistedEmbedding import com.twitter.search.common.file.{AbstractFile, LocalFile} import com.twitter.util.FuturePool import java.io.File /** * @param factory creates a BruteForceIndex from the arguments. This is only exposed for testing. * If for some reason you pass this arg in make sure that it eagerly consumes the * iterator. If you don't you might close the input stream that the iterator is * using. * @tparam T the id of the embeddings */ class BruteForceDeserialization[T, D <: Distance[D]] @VisibleForTesting private[brute_force] ( metric: Metric[D], embeddingInjection: PersistedEmbeddingInjection[T], futurePool: FuturePool, thriftIteratorIO: ThriftIteratorIO[PersistedEmbedding], factory: (Metric[D], FuturePool, Iterator[EntityEmbedding[T]]) => BruteForceIndex[T, D]) extends QueryableDeserialization[T, BruteForceRuntimeParams.type, D, BruteForceIndex[T, D]] { import BruteForceIndex._ def this( metric: Metric[D], embeddingInjection: PersistedEmbeddingInjection[T], futurePool: FuturePool, thriftIteratorIO: ThriftIteratorIO[PersistedEmbedding] ) = { this( metric, embeddingInjection, futurePool, thriftIteratorIO, factory = BruteForceIndex.apply[T, D] ) } override def fromDirectory( serializationDirectory: AbstractFile ): BruteForceIndex[T, D] = { val file = File.createTempFile(DataFileName, "tmp") file.deleteOnExit() val temp = new LocalFile(file) val dataFile = serializationDirectory.getChild(DataFileName) dataFile.copyTo(temp) val inputStream = temp.getByteSource.openBufferedStream() try { val iterator: Iterator[PersistedEmbedding] = thriftIteratorIO.fromInputStream(inputStream) val embeddings = iterator.map { thriftEmbedding => embeddingInjection.invert(thriftEmbedding).get } factory(metric, futurePool, embeddings) } finally { inputStream.close() temp.delete() } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/brute_force/BruteForceIndex.scala ================================================ package com.twitter.ann.brute_force import com.twitter.ann.common.Appendable import com.twitter.ann.common.Distance import com.twitter.ann.common.EmbeddingType._ import com.twitter.ann.common.EntityEmbedding import com.twitter.ann.common.IndexOutputFile import com.twitter.ann.common.Metric import com.twitter.ann.common.NeighborWithDistance import com.twitter.ann.common.Queryable import com.twitter.ann.common.RuntimeParams import com.twitter.ann.common.Serialization import com.twitter.ann.serialization.PersistedEmbeddingInjection import com.twitter.ann.serialization.ThriftIteratorIO import com.twitter.ann.serialization.thriftscala.PersistedEmbedding import com.twitter.search.common.file.AbstractFile import com.twitter.util.Future import com.twitter.util.FuturePool import java.util.concurrent.ConcurrentLinkedQueue import org.apache.beam.sdk.io.fs.ResourceId import scala.collection.JavaConverters._ import scala.collection.mutable object BruteForceRuntimeParams extends RuntimeParams object BruteForceIndex { val DataFileName = "BruteForceFileData" def apply[T, D <: Distance[D]]( metric: Metric[D], futurePool: FuturePool, initialEmbeddings: Iterator[EntityEmbedding[T]] = Iterator() ): BruteForceIndex[T, D] = { val linkedQueue = new ConcurrentLinkedQueue[EntityEmbedding[T]] initialEmbeddings.foreach(embedding => linkedQueue.add(embedding)) new BruteForceIndex(metric, futurePool, linkedQueue) } } class BruteForceIndex[T, D <: Distance[D]] private ( metric: Metric[D], futurePool: FuturePool, // visible for serialization private[brute_force] val linkedQueue: ConcurrentLinkedQueue[EntityEmbedding[T]]) extends Appendable[T, BruteForceRuntimeParams.type, D] with Queryable[T, BruteForceRuntimeParams.type, D] { override def append(embedding: EntityEmbedding[T]): Future[Unit] = { futurePool { linkedQueue.add(embedding) } } override def toQueryable: Queryable[T, BruteForceRuntimeParams.type, D] = this override def query( embedding: EmbeddingVector, numOfNeighbours: Int, runtimeParams: BruteForceRuntimeParams.type ): Future[List[T]] = { queryWithDistance(embedding, numOfNeighbours, runtimeParams).map { neighborsWithDistance => neighborsWithDistance.map(_.neighbor) } } override def queryWithDistance( embedding: EmbeddingVector, numOfNeighbours: Int, runtimeParams: BruteForceRuntimeParams.type ): Future[List[NeighborWithDistance[T, D]]] = { futurePool { // Use the reverse ordering so that we can call dequeue to remove the largest element. val ordering = Ordering.by[NeighborWithDistance[T, D], D](_.distance) val priorityQueue = new mutable.PriorityQueue[NeighborWithDistance[T, D]]()(ordering) linkedQueue .iterator() .asScala .foreach { entity => val neighborWithDistance = NeighborWithDistance(entity.id, metric.distance(entity.embedding, embedding)) priorityQueue.+=(neighborWithDistance) if (priorityQueue.size > numOfNeighbours) { priorityQueue.dequeue() } } val reverseList: List[NeighborWithDistance[T, D]] = priorityQueue.dequeueAll reverseList.reverse } } } object SerializableBruteForceIndex { def apply[T, D <: Distance[D]]( metric: Metric[D], futurePool: FuturePool, embeddingInjection: PersistedEmbeddingInjection[T], thriftIteratorIO: ThriftIteratorIO[PersistedEmbedding] ): SerializableBruteForceIndex[T, D] = { val bruteForceIndex = BruteForceIndex[T, D](metric, futurePool) new SerializableBruteForceIndex(bruteForceIndex, embeddingInjection, thriftIteratorIO) } } /** * This is a class that wrapps a BruteForceIndex and provides a method for serialization. * * @param bruteForceIndex all queries and updates are sent to this index. * @param embeddingInjection injection that can convert embeddings to thrift embeddings. * @param thriftIteratorIO class that provides a way to write PersistedEmbeddings to disk */ class SerializableBruteForceIndex[T, D <: Distance[D]]( bruteForceIndex: BruteForceIndex[T, D], embeddingInjection: PersistedEmbeddingInjection[T], thriftIteratorIO: ThriftIteratorIO[PersistedEmbedding]) extends Appendable[T, BruteForceRuntimeParams.type, D] with Queryable[T, BruteForceRuntimeParams.type, D] with Serialization { import BruteForceIndex._ override def append(entity: EntityEmbedding[T]): Future[Unit] = bruteForceIndex.append(entity) override def toQueryable: Queryable[T, BruteForceRuntimeParams.type, D] = this override def query( embedding: EmbeddingVector, numOfNeighbours: Int, runtimeParams: BruteForceRuntimeParams.type ): Future[List[T]] = bruteForceIndex.query(embedding, numOfNeighbours, runtimeParams) override def queryWithDistance( embedding: EmbeddingVector, numOfNeighbours: Int, runtimeParams: BruteForceRuntimeParams.type ): Future[List[NeighborWithDistance[T, D]]] = bruteForceIndex.queryWithDistance(embedding, numOfNeighbours, runtimeParams) override def toDirectory(serializationDirectory: ResourceId): Unit = { toDirectory(new IndexOutputFile(serializationDirectory)) } override def toDirectory(serializationDirectory: AbstractFile): Unit = { toDirectory(new IndexOutputFile(serializationDirectory)) } private def toDirectory(serializationDirectory: IndexOutputFile): Unit = { val outputStream = serializationDirectory.createFile(DataFileName).getOutputStream() val thriftEmbeddings = bruteForceIndex.linkedQueue.iterator().asScala.map { embedding => embeddingInjection(embedding) } try { thriftIteratorIO.toOutputStream(thriftEmbeddings, outputStream) } finally { outputStream.close() } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/AnnInjections.scala ================================================ package com.twitter.ann.common import com.twitter.bijection.{Bijection, Injection} // Class providing commonly used injections that can be used directly with ANN apis. // Injection prefixed with `J` can be used in java directly with ANN apis. object AnnInjections { val LongInjection: Injection[Long, Array[Byte]] = Injection.long2BigEndian def StringInjection: Injection[String, Array[Byte]] = Injection.utf8 def IntInjection: Injection[Int, Array[Byte]] = Injection.int2BigEndian val JLongInjection: Injection[java.lang.Long, Array[Byte]] = Bijection.long2Boxed .asInstanceOf[Bijection[Long, java.lang.Long]] .inverse .andThen(LongInjection) val JStringInjection: Injection[java.lang.String, Array[Byte]] = StringInjection val JIntInjection: Injection[java.lang.Integer, Array[Byte]] = Bijection.int2Boxed .asInstanceOf[Bijection[Int, java.lang.Integer]] .inverse .andThen(IntInjection) } ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/Api.scala ================================================ package com.twitter.ann.common import com.twitter.ann.common.EmbeddingType.EmbeddingVector import com.twitter.ml.api.embedding.Embedding import com.twitter.ml.api.embedding.EmbeddingMath import com.twitter.ml.api.embedding.EmbeddingSerDe import com.twitter.util.Future object EmbeddingType { type EmbeddingVector = Embedding[Float] val embeddingSerDe = EmbeddingSerDe.apply[Float] private[common] val math = EmbeddingMath.Float } /** * Typed entity with an embedding associated with it. * @param id : Unique Id for an entity. * @param embedding : Embedding/Vector of an entity. * @tparam T: Type of id. */ case class EntityEmbedding[T](id: T, embedding: EmbeddingVector) // Query interface for ANN trait Queryable[T, P <: RuntimeParams, D <: Distance[D]] { /** * ANN query for ids. * @param embedding: Embedding/Vector to be queried with. * @param numOfNeighbors: Number of neighbours to be queried for. * @param runtimeParams: Runtime params associated with index to control accuracy/latency etc. * @return List of approximate nearest neighbour ids. */ def query( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P ): Future[List[T]] /** * ANN query for ids with distance. * @param embedding: Embedding/Vector to be queried with. * @param numOfNeighbors: Number of neighbours to be queried for. * @param runtimeParams: Runtime params associated with index to control accuracy/latency etc. * @return List of approximate nearest neighbour ids with distance from the query embedding. */ def queryWithDistance( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P ): Future[List[NeighborWithDistance[T, D]]] } // Query interface for ANN over indexes that are grouped trait QueryableGrouped[T, P <: RuntimeParams, D <: Distance[D]] extends Queryable[T, P, D] { /** * ANN query for ids. * @param embedding: Embedding/Vector to be queried with. * @param numOfNeighbors: Number of neighbours to be queried for. * @param runtimeParams: Runtime params associated with index to control accuracy/latency etc. * @param key: Optional key to lookup specific ANN index and perform query there * @return List of approximate nearest neighbour ids. */ def query( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P, key: Option[String] ): Future[List[T]] /** * ANN query for ids with distance. * @param embedding: Embedding/Vector to be queried with. * @param numOfNeighbors: Number of neighbours to be queried for. * @param runtimeParams: Runtime params associated with index to control accuracy/latency etc. * @param key: Optional key to lookup specific ANN index and perform query there * @return List of approximate nearest neighbour ids with distance from the query embedding. */ def queryWithDistance( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P, key: Option[String] ): Future[List[NeighborWithDistance[T, D]]] } /** * Runtime params associated with index to control accuracy/latency etc while querying. */ trait RuntimeParams {} /** * ANN query result with distance. * @param neighbor : Id of the neighbours * @param distance: Distance of neighbour from query ex: D: CosineDistance, L2Distance, InnerProductDistance */ case class NeighborWithDistance[T, D <: Distance[D]](neighbor: T, distance: D) /** * ANN query result with seed entity for which this neighbor was provided. * @param seed: Seed Id for which ann query was called * @param neighbor : Id of the neighbours */ case class NeighborWithSeed[T1, T2](seed: T1, neighbor: T2) /** * ANN query result with distance with seed entity for which this neighbor was provided. * @param seed: Seed Id for which ann query was called * @param neighbor : Id of the neighbours * @param distance: Distance of neighbour from query ex: D: CosineDistance, L2Distance, InnerProductDistance */ case class NeighborWithDistanceWithSeed[T1, T2, D <: Distance[D]]( seed: T1, neighbor: T2, distance: D) trait RawAppendable[P <: RuntimeParams, D <: Distance[D]] { /** * Append an embedding in an index. * @param embedding: Embedding/Vector * @return Future of long id associated with embedding autogenerated. */ def append(embedding: EmbeddingVector): Future[Long] /** * Convert an Appendable to Queryable interface to query an index. */ def toQueryable: Queryable[Long, P, D] } // Index building interface for ANN. trait Appendable[T, P <: RuntimeParams, D <: Distance[D]] { /** * Append an entity with embedding in an index. * @param entity: Entity with its embedding */ def append(entity: EntityEmbedding[T]): Future[Unit] /** * Convert an Appendable to Queryable interface to query an index. */ def toQueryable: Queryable[T, P, D] } // Updatable index interface for ANN. trait Updatable[T] { def update(entity: EntityEmbedding[T]): Future[Unit] } ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/guava", "3rdparty/jvm/com/twitter/bijection:core", "3rdparty/jvm/com/twitter/storehaus:core", "3rdparty/jvm/org/apache/beam:beam-sdks-java-io-google-cloud-platform", "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", "finatra/inject/inject-mdc/src/main/scala", "mediaservices/commons/src/main/scala:futuretracker", "src/java/com/twitter/search/common/file", "src/scala/com/twitter/ml/api/embedding", "stitch/stitch-core", ], exports = [ "3rdparty/jvm/com/twitter/bijection:core", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/EmbeddingProducer.scala ================================================ package com.twitter.ann.common import com.twitter.stitch.Stitch trait EmbeddingProducer[T] { /** * Produce an embedding from type T. Implementations of this could do a lookup from an id to an * embedding. Or they could run a deep model on features that output and embedding. * @return An embedding Stitch. See go/stitch for details on how to use the Stitch API. */ def produceEmbedding(input: T): Stitch[Option[EmbeddingType.EmbeddingVector]] } ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/IndexOutputFile.scala ================================================ package com.twitter.ann.common import com.google.common.io.ByteStreams import com.twitter.ann.common.thriftscala.AnnIndexMetadata import com.twitter.mediaservices.commons.codec.ArrayByteBufferCodec import com.twitter.mediaservices.commons.codec.ThriftByteBufferCodec import com.twitter.search.common.file.AbstractFile import java.io.IOException import java.io.InputStream import java.io.OutputStream import java.nio.channels.Channels import org.apache.beam.sdk.io.FileSystems import org.apache.beam.sdk.io.fs.MoveOptions import org.apache.beam.sdk.io.fs.ResolveOptions import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions import org.apache.beam.sdk.io.fs.ResourceId import org.apache.beam.sdk.util.MimeTypes import org.apache.hadoop.io.IOUtils import scala.collection.JavaConverters._ /** * This class creates a wrapper around GCS filesystem and HDFS filesystem for the index * generation job. It implements the basic methods required by the index generation job and hides * the logic around handling HDFS vs GCS. */ class IndexOutputFile(val abstractFile: AbstractFile, val resourceId: ResourceId) { // Success file name private val SUCCESS_FILE = "_SUCCESS" private val INDEX_METADATA_FILE = "ANN_INDEX_METADATA" private val MetadataCodec = new ThriftByteBufferCodec[AnnIndexMetadata](AnnIndexMetadata) /** * Constructor for ResourceId. This is used for GCS filesystem * @param resourceId */ def this(resourceId: ResourceId) = { this(null, resourceId) } /** * Constructor for AbstractFile. This is used for HDFS and local filesystem * @param abstractFile */ def this(abstractFile: AbstractFile) = { this(abstractFile, null) } /** * Returns true if this instance is around an AbstractFile. * @return */ def isAbstractFile(): Boolean = { abstractFile != null } /** * Creates a _SUCCESS file in the current directory. */ def createSuccessFile(): Unit = { if (isAbstractFile()) { abstractFile.createSuccessFile() } else { val successFile = resourceId.resolve(SUCCESS_FILE, ResolveOptions.StandardResolveOptions.RESOLVE_FILE) val successWriterChannel = FileSystems.create(successFile, MimeTypes.BINARY) successWriterChannel.close() } } /** * Returns whether the current instance represents a directory * @return True if the current instance is a directory */ def isDirectory(): Boolean = { if (isAbstractFile()) { abstractFile.isDirectory } else { resourceId.isDirectory } } /** * Return the current path of the file represented by the current instance * @return The path string of the file/directory */ def getPath(): String = { if (isAbstractFile()) { abstractFile.getPath.toString } else { if (resourceId.isDirectory) { resourceId.getCurrentDirectory.toString } else { resourceId.getCurrentDirectory.toString + resourceId.getFilename } } } /** * Creates a new file @param fileName in the current directory. * @param fileName * @return A new file inside the current directory */ def createFile(fileName: String): IndexOutputFile = { if (isAbstractFile()) { // AbstractFile treats files and directories the same way. Hence, not checking for directory // here. new IndexOutputFile(abstractFile.getChild(fileName)) } else { if (!resourceId.isDirectory) { // If this is not a directory, throw exception. throw new IllegalArgumentException(getPath() + " is not a directory.") } new IndexOutputFile( resourceId.resolve(fileName, ResolveOptions.StandardResolveOptions.RESOLVE_FILE)) } } /** * Creates a new directory @param directoryName in the current directory. * @param directoryName * @return A new directory inside the current directory */ def createDirectory(directoryName: String): IndexOutputFile = { if (isAbstractFile()) { // AbstractFile treats files and directories the same way. Hence, not checking for directory // here. val dir = abstractFile.getChild(directoryName) dir.mkdirs() new IndexOutputFile(dir) } else { if (!resourceId.isDirectory) { // If this is not a directory, throw exception. throw new IllegalArgumentException(getPath() + " is not a directory.") } val newResourceId = resourceId.resolve(directoryName, ResolveOptions.StandardResolveOptions.RESOLVE_DIRECTORY) // Create a tmp file and delete in order to trigger directory creation val tmpFile = newResourceId.resolve("tmp", ResolveOptions.StandardResolveOptions.RESOLVE_FILE) val tmpWriterChannel = FileSystems.create(tmpFile, MimeTypes.BINARY) tmpWriterChannel.close() FileSystems.delete(List(tmpFile).asJava, MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES) new IndexOutputFile(newResourceId) } } def getChild(fileName: String, isDirectory: Boolean = false): IndexOutputFile = { if (isAbstractFile()) { new IndexOutputFile(abstractFile.getChild(fileName)) } else { val resolveOption = if (isDirectory) { StandardResolveOptions.RESOLVE_DIRECTORY } else { StandardResolveOptions.RESOLVE_FILE } new IndexOutputFile(resourceId.resolve(fileName, resolveOption)) } } /** * Returns an OutputStream for the underlying file. * Note: Close the OutputStream after writing * @return */ def getOutputStream(): OutputStream = { if (isAbstractFile()) { abstractFile.getByteSink.openStream() } else { if (resourceId.isDirectory) { // If this is a directory, throw exception. throw new IllegalArgumentException(getPath() + " is a directory.") } val writerChannel = FileSystems.create(resourceId, MimeTypes.BINARY) Channels.newOutputStream(writerChannel) } } /** * Returns an InputStream for the underlying file. * Note: Close the InputStream after reading * @return */ def getInputStream(): InputStream = { if (isAbstractFile()) { abstractFile.getByteSource.openStream() } else { if (resourceId.isDirectory) { // If this is a directory, throw exception. throw new IllegalArgumentException(getPath() + " is a directory.") } val readChannel = FileSystems.open(resourceId) Channels.newInputStream(readChannel) } } /** * Copies content from the srcIn into the current file. * @param srcIn */ def copyFrom(srcIn: InputStream): Unit = { val out = getOutputStream() try { IOUtils.copyBytes(srcIn, out, 4096) out.close() } catch { case ex: IOException => IOUtils.closeStream(out); throw ex; } } def writeIndexMetadata(annIndexMetadata: AnnIndexMetadata): Unit = { val out = createFile(INDEX_METADATA_FILE).getOutputStream() val bytes = ArrayByteBufferCodec.decode(MetadataCodec.encode(annIndexMetadata)) out.write(bytes) out.close() } def loadIndexMetadata(): AnnIndexMetadata = { val in = ByteStreams.toByteArray(getInputStream()) MetadataCodec.decode(ArrayByteBufferCodec.encode(in)) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/IndexTransformer.scala ================================================ package com.twitter.ann.common import com.twitter.ann.common.EmbeddingType.EmbeddingVector import com.twitter.storehaus.{ReadableStore, Store} import com.twitter.util.Future // Utility to transform raw index to typed index using Store object IndexTransformer { /** * Transform a long type queryable index to Typed queryable index * @param index: Raw Queryable index * @param store: Readable store to provide mappings between Long and T * @tparam T: Type to transform to * @tparam P: Runtime params * @return Queryable index typed on T */ def transformQueryable[T, P <: RuntimeParams, D <: Distance[D]]( index: Queryable[Long, P, D], store: ReadableStore[Long, T] ): Queryable[T, P, D] = { new Queryable[T, P, D] { override def query( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P ): Future[List[T]] = { val neighbors = index.query(embedding, numOfNeighbors, runtimeParams) neighbors .flatMap(nn => { val ids = nn.map(id => store.get(id).map(_.get)) Future .collect(ids) .map(_.toList) }) } override def queryWithDistance( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P ): Future[List[NeighborWithDistance[T, D]]] = { val neighbors = index.queryWithDistance(embedding, numOfNeighbors, runtimeParams) neighbors .flatMap(nn => { val ids = nn.map(obj => store.get(obj.neighbor).map(id => NeighborWithDistance(id.get, obj.distance))) Future .collect(ids) .map(_.toList) }) } } } /** * Transform a long type appendable index to Typed appendable index * @param index: Raw Appendable index * @param store: Writable store to store mappings between Long and T * @tparam T: Type to transform to * @return Appendable index typed on T */ def transformAppendable[T, P <: RuntimeParams, D <: Distance[D]]( index: RawAppendable[P, D], store: Store[Long, T] ): Appendable[T, P, D] = { new Appendable[T, P, D]() { override def append(entity: EntityEmbedding[T]): Future[Unit] = { index .append(entity.embedding) .flatMap(id => store.put((id, Some(entity.id)))) } override def toQueryable: Queryable[T, P, D] = { transformQueryable(index.toQueryable, store) } } } /** * Transform a long type appendable and queryable index to Typed appendable and queryable index * @param index: Raw Appendable and queryable index * @param store: Store to provide/store mappings between Long and T * @tparam T: Type to transform to * @tparam Index: Index * @return Appendable and queryable index typed on T */ def transform1[ Index <: RawAppendable[P, D] with Queryable[Long, P, D], T, P <: RuntimeParams, D <: Distance[D] ]( index: Index, store: Store[Long, T] ): Queryable[T, P, D] with Appendable[T, P, D] = { val queryable = transformQueryable(index, store) val appendable = transformAppendable(index, store) new Queryable[T, P, D] with Appendable[T, P, D] { override def query( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P ) = queryable.query(embedding, numOfNeighbors, runtimeParams) override def queryWithDistance( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P ) = queryable.queryWithDistance(embedding, numOfNeighbors, runtimeParams) override def append(entity: EntityEmbedding[T]) = appendable.append(entity) override def toQueryable: Queryable[T, P, D] = appendable.toQueryable } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/MemoizedInEpochs.scala ================================================ package com.twitter.ann.common import com.twitter.util.Return import com.twitter.util.Throw import com.twitter.util.Try import com.twitter.util.logging.Logging // Memoization with a twist // New epoch reuse K:V pairs from previous and recycle everything else class MemoizedInEpochs[K, V](f: K => Try[V]) extends Logging { private var memoizedCalls: Map[K, V] = Map.empty def epoch(keys: Seq[K]): Seq[V] = { val newSet = keys.toSet val keysToBeComputed = newSet.diff(memoizedCalls.keySet) val computedKeysAndValues = keysToBeComputed.map { key => info(s"Memoize ${key}") (key, f(key)) } val keysAndValuesAfterFilteringFailures = computedKeysAndValues .flatMap { case (key, Return(value)) => Some((key, value)) case (key, Throw(e)) => warn(s"Calling f for ${key} has failed", e) None } val keysReusedFromLastEpoch = memoizedCalls.filterKeys(newSet.contains) memoizedCalls = keysReusedFromLastEpoch ++ keysAndValuesAfterFilteringFailures debug(s"Final memoization is ${memoizedCalls.keys.mkString(", ")}") keys.flatMap(memoizedCalls.get) } def currentEpochKeys: Set[K] = memoizedCalls.keySet } ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/Metric.scala ================================================ package com.twitter.ann.common import com.google.common.collect.ImmutableBiMap import com.twitter.ann.common.EmbeddingType._ import com.twitter.ann.common.thriftscala.DistanceMetric import com.twitter.ann.common.thriftscala.{CosineDistance => ServiceCosineDistance} import com.twitter.ann.common.thriftscala.{Distance => ServiceDistance} import com.twitter.ann.common.thriftscala.{InnerProductDistance => ServiceInnerProductDistance} import com.twitter.ann.common.thriftscala.{EditDistance => ServiceEditDistance} import com.twitter.ann.common.thriftscala.{L2Distance => ServiceL2Distance} import com.twitter.bijection.Injection import scala.util.Failure import scala.util.Success import scala.util.Try // Ann distance metrics trait Distance[D] extends Any with Ordered[D] { def distance: Float } case class L2Distance(distance: Float) extends AnyVal with Distance[L2Distance] { override def compare(that: L2Distance): Int = Ordering.Float.compare(this.distance, that.distance) } case class CosineDistance(distance: Float) extends AnyVal with Distance[CosineDistance] { override def compare(that: CosineDistance): Int = Ordering.Float.compare(this.distance, that.distance) } case class InnerProductDistance(distance: Float) extends AnyVal with Distance[InnerProductDistance] { override def compare(that: InnerProductDistance): Int = Ordering.Float.compare(this.distance, that.distance) } case class EditDistance(distance: Float) extends AnyVal with Distance[EditDistance] { override def compare(that: EditDistance): Int = Ordering.Float.compare(this.distance, that.distance) } object Metric { private[this] val thriftMetricMapping = ImmutableBiMap.of( L2, DistanceMetric.L2, Cosine, DistanceMetric.Cosine, InnerProduct, DistanceMetric.InnerProduct, Edit, DistanceMetric.EditDistance ) def fromThrift(metric: DistanceMetric): Metric[_ <: Distance[_]] = { thriftMetricMapping.inverse().get(metric) } def toThrift(metric: Metric[_ <: Distance[_]]): DistanceMetric = { thriftMetricMapping.get(metric) } def fromString(metricName: String): Metric[_ <: Distance[_]] with Injection[_, ServiceDistance] = { metricName match { case "Cosine" => Cosine case "L2" => L2 case "InnerProduct" => InnerProduct case "EditDistance" => Edit case _ => throw new IllegalArgumentException(s"No Metric with the name $metricName") } } } sealed trait Metric[D <: Distance[D]] { def distance( embedding1: EmbeddingVector, embedding2: EmbeddingVector ): D def absoluteDistance( embedding1: EmbeddingVector, embedding2: EmbeddingVector ): Float def fromAbsoluteDistance(distance: Float): D } case object L2 extends Metric[L2Distance] with Injection[L2Distance, ServiceDistance] { override def distance( embedding1: EmbeddingVector, embedding2: EmbeddingVector ): L2Distance = { fromAbsoluteDistance(MetricUtil.l2distance(embedding1, embedding2).toFloat) } override def fromAbsoluteDistance(distance: Float): L2Distance = { L2Distance(distance) } override def absoluteDistance( embedding1: EmbeddingVector, embedding2: EmbeddingVector ): Float = distance(embedding1, embedding2).distance override def apply(scalaDistance: L2Distance): ServiceDistance = { ServiceDistance.L2Distance(ServiceL2Distance(scalaDistance.distance)) } override def invert(serviceDistance: ServiceDistance): Try[L2Distance] = { serviceDistance match { case ServiceDistance.L2Distance(l2Distance) => Success(L2Distance(l2Distance.distance.toFloat)) case distance => Failure(new IllegalArgumentException(s"Expected an l2 distance but got $distance")) } } } case object Cosine extends Metric[CosineDistance] with Injection[CosineDistance, ServiceDistance] { override def distance( embedding1: EmbeddingVector, embedding2: EmbeddingVector ): CosineDistance = { fromAbsoluteDistance(1 - MetricUtil.cosineSimilarity(embedding1, embedding2)) } override def fromAbsoluteDistance(distance: Float): CosineDistance = { CosineDistance(distance) } override def absoluteDistance( embedding1: EmbeddingVector, embedding2: EmbeddingVector ): Float = distance(embedding1, embedding2).distance override def apply(scalaDistance: CosineDistance): ServiceDistance = { ServiceDistance.CosineDistance(ServiceCosineDistance(scalaDistance.distance)) } override def invert(serviceDistance: ServiceDistance): Try[CosineDistance] = { serviceDistance match { case ServiceDistance.CosineDistance(cosineDistance) => Success(CosineDistance(cosineDistance.distance.toFloat)) case distance => Failure(new IllegalArgumentException(s"Expected a cosine distance but got $distance")) } } } case object InnerProduct extends Metric[InnerProductDistance] with Injection[InnerProductDistance, ServiceDistance] { override def distance( embedding1: EmbeddingVector, embedding2: EmbeddingVector ): InnerProductDistance = { fromAbsoluteDistance(1 - MetricUtil.dot(embedding1, embedding2)) } override def fromAbsoluteDistance(distance: Float): InnerProductDistance = { InnerProductDistance(distance) } override def absoluteDistance( embedding1: EmbeddingVector, embedding2: EmbeddingVector ): Float = distance(embedding1, embedding2).distance override def apply(scalaDistance: InnerProductDistance): ServiceDistance = { ServiceDistance.InnerProductDistance(ServiceInnerProductDistance(scalaDistance.distance)) } override def invert( serviceDistance: ServiceDistance ): Try[InnerProductDistance] = { serviceDistance match { case ServiceDistance.InnerProductDistance(cosineDistance) => Success(InnerProductDistance(cosineDistance.distance.toFloat)) case distance => Failure( new IllegalArgumentException(s"Expected a inner product distance but got $distance") ) } } } case object Edit extends Metric[EditDistance] with Injection[EditDistance, ServiceDistance] { private def intDistance( embedding1: EmbeddingVector, embedding2: EmbeddingVector, pos1: Int, pos2: Int, precomputedDistances: scala.collection.mutable.Map[(Int, Int), Int] ): Int = { // return the remaining characters of other String if (pos1 == 0) return pos2 if (pos2 == 0) return pos1 // To check if the recursive tree // for given n & m has already been executed precomputedDistances.getOrElse( (pos1, pos2), { // We might want to change this so that capitals are considered the same. // Also maybe some characters that look similar should also be the same. val computed = if (embedding1(pos1 - 1) == embedding2(pos2 - 1)) { intDistance(embedding1, embedding2, pos1 - 1, pos2 - 1, precomputedDistances) } else { // If characters are nt equal, we need to // find the minimum cost out of all 3 operations. val insert = intDistance(embedding1, embedding2, pos1, pos2 - 1, precomputedDistances) val del = intDistance(embedding1, embedding2, pos1 - 1, pos2, precomputedDistances) val replace = intDistance(embedding1, embedding2, pos1 - 1, pos2 - 1, precomputedDistances) 1 + Math.min(insert, Math.min(del, replace)) } precomputedDistances.put((pos1, pos2), computed) computed } ) } override def distance( embedding1: EmbeddingVector, embedding2: EmbeddingVector ): EditDistance = { val editDistance = intDistance( embedding1, embedding2, embedding1.length, embedding2.length, scala.collection.mutable.Map[(Int, Int), Int]() ) EditDistance(editDistance) } override def fromAbsoluteDistance(distance: Float): EditDistance = { EditDistance(distance.toInt) } override def absoluteDistance( embedding1: EmbeddingVector, embedding2: EmbeddingVector ): Float = distance(embedding1, embedding2).distance override def apply(scalaDistance: EditDistance): ServiceDistance = { ServiceDistance.EditDistance(ServiceEditDistance(scalaDistance.distance.toInt)) } override def invert( serviceDistance: ServiceDistance ): Try[EditDistance] = { serviceDistance match { case ServiceDistance.EditDistance(cosineDistance) => Success(EditDistance(cosineDistance.distance.toFloat)) case distance => Failure( new IllegalArgumentException(s"Expected a inner product distance but got $distance") ) } } } object MetricUtil { private[ann] def dot( embedding1: EmbeddingVector, embedding2: EmbeddingVector ): Float = { math.dotProduct(embedding1, embedding2) } private[ann] def l2distance( embedding1: EmbeddingVector, embedding2: EmbeddingVector ): Double = { math.l2Distance(embedding1, embedding2) } private[ann] def cosineSimilarity( embedding1: EmbeddingVector, embedding2: EmbeddingVector ): Float = { math.cosineSimilarity(embedding1, embedding2).toFloat } private[ann] def norm( embedding: EmbeddingVector ): EmbeddingVector = { math.normalize(embedding) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/QueryableById.scala ================================================ package com.twitter.ann.common import com.twitter.stitch.Stitch /** * This is a trait that allows you to query for nearest neighbors given an arbitrary type T1. This is * in contrast to a regular com.twitter.ann.common.Appendable, which takes an embedding as the input * argument. * * This interface uses the Stitch API for batching. See go/stitch for details on how to use it. * * @tparam T1 type of the query. * @tparam T2 type of the result. * @tparam P runtime parameters supported by the index. * @tparam D distance function used in the index. */ trait QueryableById[T1, T2, P <: RuntimeParams, D <: Distance[D]] { def queryById( id: T1, numOfNeighbors: Int, runtimeParams: P ): Stitch[List[T2]] def queryByIdWithDistance( id: T1, numOfNeighbors: Int, runtimeParams: P ): Stitch[List[NeighborWithDistance[T2, D]]] def batchQueryById( ids: Seq[T1], numOfNeighbors: Int, runtimeParams: P ): Stitch[List[NeighborWithSeed[T1, T2]]] def batchQueryWithDistanceById( ids: Seq[T1], numOfNeighbors: Int, runtimeParams: P ): Stitch[List[NeighborWithDistanceWithSeed[T1, T2, D]]] } ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/QueryableByIdImplementation.scala ================================================ package com.twitter.ann.common import com.twitter.stitch.Stitch /** * Implementation of QueryableById that composes an EmbeddingProducer and a Queryable so that we * can get nearest neighbors given an id of type T1 * @param embeddingProducer provides an embedding given an id. * @param queryable provides a list of neighbors given an embedding. * @tparam T1 type of the query. * @tparam T2 type of the result. * @tparam P runtime parameters supported by the index. * @tparam D distance function used in the index. */ class QueryableByIdImplementation[T1, T2, P <: RuntimeParams, D <: Distance[D]]( embeddingProducer: EmbeddingProducer[T1], queryable: Queryable[T2, P, D]) extends QueryableById[T1, T2, P, D] { override def queryById( id: T1, numOfNeighbors: Int, runtimeParams: P ): Stitch[List[T2]] = { embeddingProducer.produceEmbedding(id).flatMap { embeddingOption => embeddingOption .map { embedding => Stitch.callFuture(queryable.query(embedding, numOfNeighbors, runtimeParams)) }.getOrElse { Stitch.value(List.empty) } } } override def queryByIdWithDistance( id: T1, numOfNeighbors: Int, runtimeParams: P ): Stitch[List[NeighborWithDistance[T2, D]]] = { embeddingProducer.produceEmbedding(id).flatMap { embeddingOption => embeddingOption .map { embedding => Stitch.callFuture(queryable.queryWithDistance(embedding, numOfNeighbors, runtimeParams)) }.getOrElse { Stitch.value(List.empty) } } } override def batchQueryById( ids: Seq[T1], numOfNeighbors: Int, runtimeParams: P ): Stitch[List[NeighborWithSeed[T1, T2]]] = { Stitch .traverse(ids) { id => embeddingProducer.produceEmbedding(id).flatMap { embeddingOption => embeddingOption .map { embedding => Stitch .callFuture(queryable.query(embedding, numOfNeighbors, runtimeParams)).map( _.map(neighbor => NeighborWithSeed(id, neighbor))) }.getOrElse { Stitch.value(List.empty) }.handle { case _ => List.empty } } }.map { _.toList.flatten } } override def batchQueryWithDistanceById( ids: Seq[T1], numOfNeighbors: Int, runtimeParams: P ): Stitch[List[NeighborWithDistanceWithSeed[T1, T2, D]]] = { Stitch .traverse(ids) { id => embeddingProducer.produceEmbedding(id).flatMap { embeddingOption => embeddingOption .map { embedding => Stitch .callFuture(queryable.queryWithDistance(embedding, numOfNeighbors, runtimeParams)) .map(_.map(neighbor => NeighborWithDistanceWithSeed(id, neighbor.neighbor, neighbor.distance))) }.getOrElse { Stitch.value(List.empty) }.handle { case _ => List.empty } } }.map { _.toList.flatten } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/QueryableOperations.scala ================================================ package com.twitter.ann.common import com.twitter.ann.common.EmbeddingType.EmbeddingVector import com.twitter.util.Future object QueryableOperations { implicit class Map[T, P <: RuntimeParams, D <: Distance[D]]( val q: Queryable[T, P, D]) { def mapRuntimeParameters(f: P => P): Queryable[T, P, D] = { new Queryable[T, P, D] { def query( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P ): Future[List[T]] = q.query(embedding, numOfNeighbors, f(runtimeParams)) def queryWithDistance( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P ): Future[List[NeighborWithDistance[T, D]]] = q.queryWithDistance(embedding, numOfNeighbors, f(runtimeParams)) } } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/ReadWriteFuturePool.scala ================================================ package com.twitter.ann.common import com.google.common.annotations.VisibleForTesting import com.twitter.util.{Future, FuturePool} trait ReadWriteFuturePool { def read[T](f: => T): Future[T] def write[T](f: => T): Future[T] } object ReadWriteFuturePool { def apply(readPool: FuturePool, writePool: FuturePool): ReadWriteFuturePool = { new ReadWriteFuturePoolANN(readPool, writePool) } def apply(commonPool: FuturePool): ReadWriteFuturePool = { new ReadWriteFuturePoolANN(commonPool, commonPool) } } @VisibleForTesting private[ann] class ReadWriteFuturePoolANN(readPool: FuturePool, writePool: FuturePool) extends ReadWriteFuturePool { def read[T](f: => T): Future[T] = { readPool.apply(f) } def write[T](f: => T): Future[T] = { writePool.apply(f) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/Serialization.scala ================================================ package com.twitter.ann.common import com.twitter.search.common.file.AbstractFile import org.apache.beam.sdk.io.fs.ResourceId /** * Interface for writing an Appendable to a directory. */ trait Serialization { def toDirectory( serializationDirectory: AbstractFile ): Unit def toDirectory( serializationDirectory: ResourceId ): Unit } /** * Interface for reading a Queryable from a directory * @tparam T the id of the embeddings * @tparam Q type of the Queryable that is deserialized. */ trait QueryableDeserialization[T, P <: RuntimeParams, D <: Distance[D], Q <: Queryable[T, P, D]] { def fromDirectory( serializationDirectory: AbstractFile ): Q } ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/ServiceClientQueryable.scala ================================================ package com.twitter.ann.common import com.twitter.ann.common.EmbeddingType._ import com.twitter.ann.common.thriftscala.{ NearestNeighborQuery, NearestNeighborResult, Distance => ServiceDistance, RuntimeParams => ServiceRuntimeParams } import com.twitter.bijection.Injection import com.twitter.finagle.Service import com.twitter.mediaservices.commons.codec.ArrayByteBufferCodec import com.twitter.util.Future class ServiceClientQueryable[T, P <: RuntimeParams, D <: Distance[D]]( service: Service[NearestNeighborQuery, NearestNeighborResult], runtimeParamInjection: Injection[P, ServiceRuntimeParams], distanceInjection: Injection[D, ServiceDistance], idInjection: Injection[T, Array[Byte]]) extends Queryable[T, P, D] { override def query( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P ): Future[List[T]] = { service .apply( NearestNeighborQuery( embeddingSerDe.toThrift(embedding), withDistance = false, runtimeParamInjection(runtimeParams), numOfNeighbors ) ) .map { result => result.nearestNeighbors.map { nearestNeighbor => idInjection.invert(ArrayByteBufferCodec.decode(nearestNeighbor.id)).get }.toList } } override def queryWithDistance( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P ): Future[List[NeighborWithDistance[T, D]]] = service .apply( NearestNeighborQuery( embeddingSerDe.toThrift(embedding), withDistance = true, runtimeParamInjection(runtimeParams), numOfNeighbors ) ) .map { result => result.nearestNeighbors.map { nearestNeighbor => NeighborWithDistance( idInjection.invert(ArrayByteBufferCodec.decode(nearestNeighbor.id)).get, distanceInjection.invert(nearestNeighbor.distance.get).get ) }.toList } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/ShardApi.scala ================================================ package com.twitter.ann.common import com.twitter.ann.common.EmbeddingType.EmbeddingVector import com.twitter.util.Future import scala.util.Random trait ShardFunction[T] { /** * Shard function to shard embedding based on total shards and embedding data. * @param shards * @param entity * @return Shard index, from 0(Inclusive) to shards(Exclusive)) */ def apply(shards: Int, entity: EntityEmbedding[T]): Int } /** * Randomly shards the embeddings based on number of total shards. */ class RandomShardFunction[T] extends ShardFunction[T] { def apply(shards: Int, entity: EntityEmbedding[T]): Int = { Random.nextInt(shards) } } /** * Sharded appendable to shard the embedding into different appendable indices * @param indices: Sequence of appendable indices * @param shardFn: Shard function to shard data into different indices * @param shards: Total shards * @tparam T: Type of id. */ class ShardedAppendable[T, P <: RuntimeParams, D <: Distance[D]]( indices: Seq[Appendable[T, P, D]], shardFn: ShardFunction[T], shards: Int) extends Appendable[T, P, D] { override def append(entity: EntityEmbedding[T]): Future[Unit] = { val shard = shardFn(shards, entity) val index = indices(shard) index.append(entity) } override def toQueryable: Queryable[T, P, D] = { new ComposedQueryable[T, P, D](indices.map(_.toQueryable)) } } /** * Composition of sequence of queryable indices, it queries all the indices, * and merges the result in memory to return the K nearest neighbours * @param indices: Sequence of queryable indices * @tparam T: Type of id * @tparam P: Type of runtime param * @tparam D: Type of distance metric */ class ComposedQueryable[T, P <: RuntimeParams, D <: Distance[D]]( indices: Seq[Queryable[T, P, D]]) extends Queryable[T, P, D] { private[this] val ordering = Ordering.by[NeighborWithDistance[T, D], D](_.distance) override def query( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P ): Future[List[T]] = { val neighbours = queryWithDistance(embedding, numOfNeighbors, runtimeParams) neighbours.map(list => list.map(nn => nn.neighbor)) } override def queryWithDistance( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P ): Future[List[NeighborWithDistance[T, D]]] = { val futures = Future.collect( indices.map(index => index.queryWithDistance(embedding, numOfNeighbors, runtimeParams)) ) futures.map { list => list.flatten .sorted(ordering) .take(numOfNeighbors) .toList } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/ShardedSerialization.scala ================================================ package com.twitter.ann.common import com.twitter.search.common.file.AbstractFile import com.twitter.search.common.file.AbstractFile.Filter import com.twitter.util.Future import org.apache.beam.sdk.io.fs.ResourceId import scala.collection.JavaConverters._ object ShardConstants { val ShardPrefix = "shard_" } /** * Serialize shards to directory * @param shards: List of shards to serialize */ class ShardedSerialization( shards: Seq[Serialization]) extends Serialization { override def toDirectory(directory: AbstractFile): Unit = { toDirectory(new IndexOutputFile(directory)) } override def toDirectory(directory: ResourceId): Unit = { toDirectory(new IndexOutputFile(directory)) } private def toDirectory(directory: IndexOutputFile): Unit = { shards.indices.foreach { shardId => val shardDirectory = directory.createDirectory(ShardConstants.ShardPrefix + shardId) val serialization = shards(shardId) if (shardDirectory.isAbstractFile) { serialization.toDirectory(shardDirectory.abstractFile) } else { serialization.toDirectory(shardDirectory.resourceId) } } } } /** * Deserialize directories containing index shards data to a composed queryable * @param deserializationFn function to deserialize a shard file to Queryable * @tparam T the id of the embeddings * @tparam P : Runtime params type * @tparam D: Distance metric type */ class ComposedQueryableDeserialization[T, P <: RuntimeParams, D <: Distance[D]]( deserializationFn: (AbstractFile) => Queryable[T, P, D]) extends QueryableDeserialization[T, P, D, Queryable[T, P, D]] { override def fromDirectory(directory: AbstractFile): Queryable[T, P, D] = { val shardDirs = directory .listFiles(new Filter { override def accept(file: AbstractFile): Boolean = file.getName.startsWith(ShardConstants.ShardPrefix) }) .asScala .toList val indices = shardDirs .map { shardDir => deserializationFn(shardDir) } new ComposedQueryable[T, P, D](indices) } } class ShardedIndexBuilderWithSerialization[T, P <: RuntimeParams, D <: Distance[D]]( shardedIndex: ShardedAppendable[T, P, D], shardedSerialization: ShardedSerialization) extends Appendable[T, P, D] with Serialization { override def append(entity: EntityEmbedding[T]): Future[Unit] = { shardedIndex.append(entity) } override def toDirectory(directory: AbstractFile): Unit = { shardedSerialization.toDirectory(directory) } override def toDirectory(directory: ResourceId): Unit = { shardedSerialization.toDirectory(directory) } override def toQueryable: Queryable[T, P, D] = { shardedIndex.toQueryable } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/common/Task.scala ================================================ package com.twitter.ann.common import com.twitter.finagle.stats.CategorizingExceptionStatsHandler import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.tracing.DefaultTracer import com.twitter.finagle.tracing.Trace import com.twitter.finagle.util.DefaultTimer import com.twitter.finagle.util.Rng import com.twitter.inject.logging.MDCKeys import com.twitter.util.Closable import com.twitter.util.Duration import com.twitter.util.Future import com.twitter.util.Time import com.twitter.util.Timer import com.twitter.util.logging.Logging import java.util.concurrent.atomic.AtomicInteger import org.slf4j.MDC /** * A Task that will be scheduled to execute periodically on every interval. If a task takes * longer than an interval to complete, it will be immediately scheduled to run. */ trait Task extends Closable { self: Logging => // Exposed if the implementation of `task` need to report failures val exnStatsHandler = new CategorizingExceptionStatsHandler(categorizer = _ => Some("failures")) protected val statsReceiver: StatsReceiver private val totalTasks = statsReceiver.counter("total") private val successfulTasks = statsReceiver.counter("success") private val taskLatency = statsReceiver.stat("latency_ms") private val activeTasks = new AtomicInteger(0) protected[common] val rng: Rng = Rng.threadLocal protected[common] val timer: Timer = DefaultTimer @volatile private var taskLoop: Future[Unit] = null /** Execute the task wih bookkeeping **/ private def run(): Future[Unit] = { totalTasks.incr() activeTasks.getAndIncrement() val start = Time.now val runningTask = // Setup a new trace root for this task. We also want logs to contain // the same trace information finatra populates for requests. // See com.twitter.finatra.thrift.filters.TraceIdMDCFilter Trace.letTracerAndNextId(DefaultTracer) { val trace = Trace() MDC.put(MDCKeys.TraceId, trace.id.traceId.toString) MDC.put(MDCKeys.TraceSampled, trace.id._sampled.getOrElse(false).toString) MDC.put(MDCKeys.TraceSpanId, trace.id.spanId.toString) info(s"starting task ${getClass.toString}") task() .onSuccess({ _ => info(s"completed task ${getClass.toString}") successfulTasks.incr() }) .onFailure({ e => warn(s"failed task. ", e) exnStatsHandler.record(statsReceiver, e) }) } runningTask.transform { _ => val elapsed = Time.now - start activeTasks.getAndDecrement() taskLatency.add(elapsed.inMilliseconds) Future .sleep(taskInterval)(timer) .before(run()) } } // Body of a task to run protected def task(): Future[Unit] // Task interval protected def taskInterval: Duration /** * Start the task after random jitter */ final def jitteredStart(): Unit = synchronized { if (taskLoop != null) { throw new RuntimeException(s"task already started") } else { val jitterNs = rng.nextLong(taskInterval.inNanoseconds) val jitter = Duration.fromNanoseconds(jitterNs) taskLoop = Future .sleep(jitter)(timer) .before(run()) } } /** * Start the task without applying any delay */ final def startImmediately(): Unit = synchronized { if (taskLoop != null) { throw new RuntimeException(s"task already started") } else { taskLoop = run() } } /** * Close the task. A closed task cannot be restarted. */ override def close(deadline: Time): Future[Unit] = { if (taskLoop != null) { taskLoop.raise(new InterruptedException("task closed")) } Future.Done } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/dataflow/offline/ANNIndexBuilderBeamJob.scala ================================================ package com.twitter.ann.dataflow.offline import com.spotify.scio.ScioContext import com.spotify.scio.ScioMetrics import com.twitter.ann.annoy.TypedAnnoyIndex import com.twitter.ann.brute_force.SerializableBruteForceIndex import com.twitter.ann.common.thriftscala.AnnIndexMetadata import com.twitter.ann.common.Distance import com.twitter.ann.common.Cosine import com.twitter.ann.common.EntityEmbedding import com.twitter.ann.common.IndexOutputFile import com.twitter.ann.common.Metric import com.twitter.ann.common.ReadWriteFuturePool import com.twitter.ann.faiss.FaissIndexer import com.twitter.ann.hnsw.TypedHnswIndex import com.twitter.ann.serialization.PersistedEmbeddingInjection import com.twitter.ann.serialization.ThriftIteratorIO import com.twitter.ann.serialization.thriftscala.PersistedEmbedding import com.twitter.ann.util.IndexBuilderUtils import com.twitter.beam.io.bigquery.BigQueryIO import com.twitter.beam.io.dal.DalObservedDatasetRegistration import com.twitter.beam.job.DateRange import com.twitter.beam.job.DateRangeOptions import com.twitter.cortex.ml.embeddings.common._ import com.twitter.ml.api.embedding.Embedding import com.twitter.ml.api.embedding.EmbeddingMath import com.twitter.ml.api.embedding.EmbeddingSerDe import com.twitter.ml.api.thriftscala.{Embedding => TEmbedding} import com.twitter.ml.featurestore.lib.EntityId import com.twitter.ml.featurestore.lib.SemanticCoreId import com.twitter.ml.featurestore.lib.TfwId import com.twitter.ml.featurestore.lib.TweetId import com.twitter.ml.featurestore.lib.UserId import com.twitter.scalding.DateOps import com.twitter.scalding.RichDate import com.twitter.scio_internal.job.ScioBeamJob import com.twitter.statebird.v2.thriftscala.{Environment => StatebirdEnvironment} import com.twitter.util.Await import com.twitter.util.FuturePool import com.twitter.wtf.beam.bq_embedding_export.BQQueryUtils import java.time.Instant import java.util.TimeZone import java.util.concurrent.Executors import org.apache.beam.sdk.io.FileSystems import org.apache.beam.sdk.io.fs.ResolveOptions import org.apache.beam.sdk.io.fs.ResourceId import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead import org.apache.beam.sdk.options.Default import org.apache.beam.sdk.options.Description import org.apache.beam.sdk.transforms.DoFn import org.apache.beam.sdk.transforms.DoFn._ import org.apache.beam.sdk.transforms.PTransform import org.apache.beam.sdk.transforms.ParDo import org.apache.beam.sdk.values.KV import org.apache.beam.sdk.values.PCollection import org.apache.beam.sdk.values.PDone import org.slf4j.Logger import org.slf4j.LoggerFactory trait ANNOptions extends DateRangeOptions { @Description("Output GCS path for the generated index") def getOutputPath(): String def setOutputPath(value: String): Unit @Description("If set, the index is grouped") @Default.Boolean(false) def getGrouped: Boolean def setGrouped(value: Boolean): Unit @Description( "If set, a segment will be registered for the provided DAL dataset module which will trigger " + "DAL registration.") @Default.Boolean(false) def getEnableDalRegistration: Boolean def setEnableDalRegistration(value: Boolean): Unit @Description( "Output GCS path for the generated index. The OutputPath should be of the format " + "'gs://user.{{user_name}}.dp.gcp.twttr.net/subDir/outputDir' and OutputDALPath will be " + "'subDir/outputDir' for this to work") def getOutputDALPath: String def setOutputDALPath(value: String): Unit @Description("Get ANN index dataset name") def getDatasetModuleName: String def setDatasetModuleName(value: String): Unit @Description("Get ANN index dataset owner role") def getDatasetOwnerRole: String def setDatasetOwnerRole(value: String): Unit @Description("If set, index is written in /") @Default.Boolean(false) def getOutputWithTimestamp: Boolean def setOutputWithTimestamp(value: Boolean): Unit @Description("File which contains a SQL query to retrieve embeddings from BQ") def getDatasetSqlPath: String def setDatasetSqlPath(value: String): Unit @Description("Dimension of embedding in the input data. See go/ann") def getDimension: Int def setDimension(value: Int): Unit @Description("The type of entity ID that is used with the embeddings. See go/ann") def getEntityKind: String def setEntityKind(value: String): Unit @Description("The kind of index you want to generate (HNSW/Annoy/Brute Force/faiss). See go/ann") def getAlgo: String def setAlgo(value: String): Unit @Description("Distance metric (InnerProduct/Cosine/L2). See go/ann") def getMetric: String def setMetric(value: String): Unit @Description("Specifies how many parallel inserts happen to the index. See go/ann") def getConcurrencyLevel: Int def setConcurrencyLevel(value: Int): Unit @Description( "Used by HNSW algo. Larger value increases build time but will give better recall. See go/ann") def getEfConstruction: Int def setEfConstruction(value: Int): Unit @Description( "Used by HNSW algo. Larger value increases the index size but will give better recall. " + "See go/ann") def getMaxM: Int def setMaxM(value: Int): Unit @Description("Used by HNSW algo. Approximate number of elements that will be indexed. See go/ann") def getExpectedElements: Int def setExpectedElements(value: Int): Unit @Description( "Used by Annoy. num_trees is provided during build time and affects the build time and the " + "index size. A larger value will give more accurate results, but larger indexes. See go/ann") def getAnnoyNumTrees: Int def setAnnoyNumTrees(value: Int): Unit @Description( "FAISS factory string determines the ANN algorithm and compression. " + "See https://github.com/facebookresearch/faiss/wiki/The-index-factory") def getFAISSFactoryString: String def setFAISSFactoryString(value: String): Unit @Description("Sample rate for training during creation of FAISS index. Default is 0.05f") @Default.Float(0.05f) def getTrainingSampleRate: Float def setTrainingSampleRate(value: Float): Unit } /** * Builds ANN index. * * The input embeddings are read from BigQuery using the input SQL query. The output from this SQL * query needs to have two columns, "entityID" [Long] and "embedding" [List[Double]] * * Output directory supported is GCS bucket */ object ANNIndexBuilderBeamJob extends ScioBeamJob[ANNOptions] { val counterNameSpace = "ANNIndexBuilderBeamJob" val LOG: Logger = LoggerFactory.getLogger(this.getClass) implicit val timeZone: TimeZone = DateOps.UTC def configurePipeline(sc: ScioContext, opts: ANNOptions): Unit = { val startDate: RichDate = RichDate(opts.interval.getStart.toDate) val endDate: RichDate = RichDate(opts.interval.getEnd.toDate) val instant = Instant.now() val out = { val base = FileSystems.matchNewResource(opts.getOutputPath, /*isDirectory=*/ true) if (opts.getOutputWithTimestamp) { base.resolve( instant.toEpochMilli.toString, ResolveOptions.StandardResolveOptions.RESOLVE_DIRECTORY) } else { base } } // Define template variables which we would like to be replaced in the corresponding sql file val templateVariables = Map( "START_DATE" -> startDate.toString(DateOps.DATETIME_HMS_WITH_DASH), "END_DATE" -> endDate.toString(DateOps.DATETIME_HMS_WITH_DASH) ) val embeddingFetchQuery = BQQueryUtils.getBQQueryFromSqlFile(opts.getDatasetSqlPath, templateVariables) val sCollection = if (opts.getGrouped) { sc.customInput( "Read grouped data from BQ", BigQueryIO .readClass[GroupedEmbeddingData]() .fromQuery(embeddingFetchQuery).usingStandardSql() .withMethod(TypedRead.Method.DIRECT_READ) ) } else { sc.customInput( "Read flat data from BQ", BigQueryIO .readClass[FlatEmbeddingData]().fromQuery(embeddingFetchQuery).usingStandardSql() .withMethod(TypedRead.Method.DIRECT_READ) ) } val processedCollection = sCollection .flatMap(transformTableRowToKeyVal) .groupBy(_.getKey) .map { case (groupName, groupValue) => Map(groupName -> groupValue.map(_.getValue)) } val annIndexMetadata = AnnIndexMetadata(timestamp = Some(instant.getEpochSecond), withGroups = Some(opts.getGrouped)) // Count the number of groups and output the ANN index metadata processedCollection.count.map(count => { val annGroupedIndexMetadata = annIndexMetadata.copy( numGroups = Some(count.intValue()) ) val indexOutDir = new IndexOutputFile(out) indexOutDir.writeIndexMetadata(annGroupedIndexMetadata) }) // Generate Index processedCollection.saveAsCustomOutput( "Serialise to Disk", OutputSink( out, opts.getAlgo.equals("faiss"), opts.getOutputDALPath, opts.getEnableDalRegistration, opts.getDatasetModuleName, opts.getDatasetOwnerRole, instant, opts.getDate(), counterNameSpace ) ) } def transformTableRowToKeyVal( data: BaseEmbeddingData ): Option[KV[String, KV[Long, TEmbedding]]] = { val transformTable = ScioMetrics.counter(counterNameSpace, "transform_table_row_to_kv") for { id <- data.entityId } yield { transformTable.inc() val groupName: String = if (data.isInstanceOf[GroupedEmbeddingData]) { (data.asInstanceOf[GroupedEmbeddingData]).groupId.get } else { "" } KV.of[String, KV[Long, TEmbedding]]( groupName, KV.of[Long, TEmbedding]( id, EmbeddingSerDe.toThrift(Embedding(data.embedding.map(_.toFloat).toArray))) ) } } case class OutputSink( outDir: ResourceId, isFaiss: Boolean, outputDALPath: String, enableDalRegistration: Boolean, datasetModuleName: String, datasetOwnerRole: String, instant: Instant, date: DateRange, counterNameSpace: String) extends PTransform[PCollection[Map[String, Iterable[KV[Long, TEmbedding]]]], PDone] { override def expand(input: PCollection[Map[String, Iterable[KV[Long, TEmbedding]]]]): PDone = { PDone.in { val dummyOutput = { if (isFaiss) { input .apply( "Build&WriteFaissANNIndex", ParDo.of(new BuildFaissANNIndex(outDir, counterNameSpace)) ) } else { input .apply( "Build&WriteANNIndex", ParDo.of(new BuildANNIndex(outDir, counterNameSpace)) ) } } if (enableDalRegistration) { input .apply( "Register DAL Dataset", DalObservedDatasetRegistration( datasetModuleName, datasetOwnerRole, outputDALPath, instant, Some(StatebirdEnvironment.Prod), Some("ANN Index Data Files")) ) .getPipeline } else { dummyOutput.getPipeline } } } } class BuildANNIndex(outDir: ResourceId, counterNameSpace: String) extends DoFn[Map[String, Iterable[KV[Long, TEmbedding]]], Unit] { def transformKeyValToEmbeddingWithEntity[T <: EntityId]( entityKind: EntityKind[T] )( keyVal: KV[Long, TEmbedding] ): EntityEmbedding[T] = { val entityId = entityKind match { case UserKind => UserId(keyVal.getKey).toThrift case TweetKind => TweetId(keyVal.getKey).toThrift case TfwKind => TfwId(keyVal.getKey).toThrift case SemanticCoreKind => SemanticCoreId(keyVal.getKey).toThrift case _ => throw new IllegalArgumentException(s"Unsupported embedding kind: $entityKind") } EntityEmbedding[T]( EntityId.fromThrift(entityId).asInstanceOf[T], EmbeddingSerDe.fromThrift(keyVal.getValue)) } @ProcessElement def processElement[T <: EntityId, D <: Distance[D]]( @Element dataGrouped: Map[String, Iterable[KV[Long, TEmbedding]]], context: ProcessContext ): Unit = { val opts = context.getPipelineOptions.as(classOf[ANNOptions]) val uncastEntityKind = EntityKind.getEntityKind(opts.getEntityKind) val entityKind = uncastEntityKind.asInstanceOf[EntityKind[T]] val transformKVtoEmbeddings = ScioMetrics.counter(counterNameSpace, "transform_kv_to_embeddings") val _ = dataGrouped.map { case (groupName, data) => val annEmbeddings = data.map { kv => transformKVtoEmbeddings.inc() transformKeyValToEmbeddingWithEntity(entityKind)(kv) } val out = { if (opts.getGrouped && groupName != "") { outDir.resolve(groupName, ResolveOptions.StandardResolveOptions.RESOLVE_DIRECTORY) } else { outDir } } LOG.info(s"Writing output to ${out}") val metric = Metric.fromString(opts.getMetric).asInstanceOf[Metric[D]] val concurrencyLevel = opts.getConcurrencyLevel val dimension = opts.getDimension val threadPool = Executors.newFixedThreadPool(concurrencyLevel) LOG.info(s"Building ANN index of type ${opts.getAlgo}") val serialization = opts.getAlgo match { case "brute_force" => val PersistedEmbeddingIO = new ThriftIteratorIO[PersistedEmbedding](PersistedEmbedding) SerializableBruteForceIndex( metric, FuturePool.apply(threadPool), new PersistedEmbeddingInjection(entityKind.byteInjection), PersistedEmbeddingIO ) case "annoy" => TypedAnnoyIndex.indexBuilder( dimension, opts.getAnnoyNumTrees, metric, entityKind.byteInjection, FuturePool.apply(threadPool) ) case "hnsw" => val efConstruction = opts.getEfConstruction val maxM = opts.getMaxM val expectedElements = opts.getExpectedElements TypedHnswIndex.serializableIndex( dimension, metric, efConstruction, maxM, expectedElements, entityKind.byteInjection, ReadWriteFuturePool(FuturePool.apply(threadPool)) ) } val future = IndexBuilderUtils.addToIndex(serialization, annEmbeddings.toSeq, concurrencyLevel) Await.result(future.map { _ => serialization.toDirectory(out) }) } } } class BuildFaissANNIndex(outDir: ResourceId, counterNameSpace: String) extends DoFn[Map[String, Iterable[KV[Long, TEmbedding]]], Unit] { @ProcessElement def processElement[D <: Distance[D]]( @Element dataGrouped: Map[String, Iterable[KV[Long, TEmbedding]]], context: ProcessContext ): Unit = { val opts = context.getPipelineOptions.as(classOf[ANNOptions]) val transformKVtoEmbeddings = ScioMetrics.counter(counterNameSpace, "transform_kv_to_embeddings") val _ = dataGrouped.map { case (groupName, data) => val out = { if (opts.getGrouped && groupName != "") { outDir.resolve(groupName, ResolveOptions.StandardResolveOptions.RESOLVE_DIRECTORY) } else { outDir } } LOG.info(s"Writing output to ${out}") val metric = Metric.fromString(opts.getMetric).asInstanceOf[Metric[D]] val maybeNormalizedPipe = data.map { kv => transformKVtoEmbeddings.inc() val embedding = EmbeddingSerDe.floatEmbeddingSerDe.fromThrift(kv.getValue) EntityEmbedding[Long]( kv.getKey, if (metric == Cosine) { EmbeddingMath.Float.normalize(embedding) } else { embedding } ) } // Generate Index FaissIndexer.buildAndWriteFaissIndex( maybeNormalizedPipe, opts.getTrainingSampleRate, opts.getFAISSFactoryString, metric, new IndexOutputFile(out)) } } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/dataflow/offline/BUILD ================================================ scala_library( name = "index_builder_lib", sources = [ "*.scala", ], tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/spotify:scio-core", "3rdparty/jvm/org/apache/beam:beam-sdks-java-core", "ann/src/main/java/com/twitter/ann/faiss", "ann/src/main/scala/com/twitter/ann/annoy", "ann/src/main/scala/com/twitter/ann/brute_force", "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/scala/com/twitter/ann/faiss", "ann/src/main/scala/com/twitter/ann/hnsw", "ann/src/main/scala/com/twitter/ann/serialization", "ann/src/main/scala/com/twitter/ann/util", "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", "beam-internal/src/main/scala/com/twitter/beam/io/bigquery", "beam-internal/src/main/scala/com/twitter/beam/io/dal", "beam-internal/src/main/scala/com/twitter/beam/job", "beam-internal/src/main/scala/com/twitter/scio_internal/runner/dataflow", "src/scala/com/twitter/cortex/ml/embeddings/common:Helpers", "src/scala/com/twitter/ml/featurestore/lib", "src/scala/com/twitter/wtf/beam/bq_embedding_export:bq_embedding_export_lib", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/dataflow/offline/BaseEmbeddingData.scala ================================================ package com.twitter.ann.dataflow.offline trait BaseEmbeddingData { val entityId: Option[Long] val embedding: Seq[Double] } ================================================ FILE: ann/src/main/scala/com/twitter/ann/dataflow/offline/FlatEmbeddingData.scala ================================================ package com.twitter.ann.dataflow.offline import com.twitter.beam.schemas.SchemaFieldName case class FlatEmbeddingData( @SchemaFieldName("entityId") entityId: Option[Long], @SchemaFieldName("embedding") embedding: Seq[Double]) extends BaseEmbeddingData ================================================ FILE: ann/src/main/scala/com/twitter/ann/dataflow/offline/GroupedEmbeddingData.scala ================================================ package com.twitter.ann.dataflow.offline import com.twitter.beam.schemas.SchemaFieldName case class GroupedEmbeddingData( @SchemaFieldName("entityId") entityId: Option[Long], @SchemaFieldName("embedding") embedding: Seq[Double], @SchemaFieldName("groupId") groupId: Option[String], ) extends BaseEmbeddingData ================================================ FILE: ann/src/main/scala/com/twitter/ann/experimental/BUILD.bazel ================================================ scala_library( name = "server", sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-only"], dependencies = [ "ann/src/main/scala/com/twitter/ann/annoy", "ann/src/main/scala/com/twitter/ann/brute_force", "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/scala/com/twitter/ann/hnsw", ], ) hadoop_binary( name = "benchmarking", basename = "benchmarking", main = "com.twitter.ann.experimental.Runner", platform = "java8", runtime_platform = "java8", tags = [ "bazel-compatible", "bazel-compatible:migrated", "bazel-only", ], dependencies = [ ":server", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/experimental/Runner.scala ================================================ package com.twitter.ann.experimental import com.twitter.ann.annoy.{AnnoyRuntimeParams, TypedAnnoyIndex} import com.twitter.ann.brute_force.{BruteForceIndex, BruteForceRuntimeParams} import com.twitter.ann.common.{Cosine, CosineDistance, EntityEmbedding, ReadWriteFuturePool} import com.twitter.ann.hnsw.{HnswParams, TypedHnswIndex} import com.twitter.bijection.Injection import com.twitter.ml.api.embedding.Embedding import com.twitter.search.common.file.LocalFile import com.twitter.util.{Await, Future, FuturePool} import java.nio.file.Files import java.util import java.util.concurrent.Executors import java.util.{Collections, Random} import scala.collection.JavaConverters._ import scala.collection.mutable object Runner { def main(args: Array[String]): Unit = { val rng = new Random() val dimen = 300 val neighbours = 20 val trainDataSetSize = 2000 val testDataSetSize = 30 // Hnsw (ef -> (time, recall)) val hnswEfConfig = new mutable.HashMap[Int, (Float, Float)] val efConstruction = 200 val maxM = 16 val threads = 24 val efSearch = Seq(20, 30, 50, 70, 100, 120) efSearch.foreach(hnswEfConfig.put(_, (0.0f, 0.0f))) // Annoy (nodes to explore -> (time, recall)) val numOfTrees = 80 val annoyConfig = new mutable.HashMap[Int, (Float, Float)] val nodesToExplore = Seq(0, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 30000, 35000, 40000, 50000) nodesToExplore.foreach(annoyConfig.put(_, (0.0f, 0.0f))) val injection = Injection.int2BigEndian val distance = Cosine val exec = Executors.newFixedThreadPool(threads) val pool = FuturePool.apply(exec) val hnswMultiThread = TypedHnswIndex.index[Int, CosineDistance]( dimen, distance, efConstruction = efConstruction, maxM = maxM, trainDataSetSize, ReadWriteFuturePool(pool) ) val bruteforce = BruteForceIndex[Int, CosineDistance](distance, pool) val annoyBuilder = TypedAnnoyIndex.indexBuilder(dimen, numOfTrees, distance, injection, FuturePool.immediatePool) val temp = new LocalFile(Files.createTempDirectory("test").toFile) println("Creating bruteforce.........") val data = Collections.synchronizedList(new util.ArrayList[EntityEmbedding[Int]]()) val bruteforceFutures = 1 to trainDataSetSize map { id => val vec = Array.fill(dimen)(rng.nextFloat() * 50) val emb = EntityEmbedding[Int](id, Embedding(vec)) data.add(emb) bruteforce.append(emb) } Await.result(Future.collect(bruteforceFutures)) println("Creating hnsw multithread test.........") val (_, multiThreadInsertion) = time { Await.result(Future.collect(data.asScala.toList.map { emb => hnswMultiThread.append(emb) })) } println("Creating annoy.........") val (_, annoyTime) = time { Await.result(Future.collect(data.asScala.toList.map(emb => annoyBuilder.append(emb)))) annoyBuilder.toDirectory(temp) } val annoyQuery = TypedAnnoyIndex.loadQueryableIndex( dimen, Cosine, injection, FuturePool.immediatePool, temp ) val hnswQueryable = hnswMultiThread.toQueryable println(s"Total train size : $trainDataSetSize") println(s"Total querySize : $testDataSetSize") println(s"Dimension : $dimen") println(s"Distance type : $distance") println(s"Annoy index creation time trees: $numOfTrees => $annoyTime ms") println( s"Hnsw multi thread creation time : $multiThreadInsertion ms efCons: $efConstruction maxM $maxM thread : $threads") println("Querying.........") var bruteForceTime = 0.0f 1 to testDataSetSize foreach { id => println("Querying id " + id) val embedding = Embedding(Array.fill(dimen)(rng.nextFloat())) val (list, timeTakenB) = time( Await .result( bruteforce.query(embedding, neighbours, BruteForceRuntimeParams)) .toSet) bruteForceTime += timeTakenB val annoyConfigCopy = annoyConfig.toMap val hnswEfConfigCopy = hnswEfConfig.toMap hnswEfConfigCopy.keys.foreach { ef => val (nn, timeTaken) = time(Await .result(hnswQueryable.query(embedding, neighbours, HnswParams(ef))) .toSet) val recall = (list.intersect(nn).size) * 1.0f / neighbours val (oldTime, oldRecall) = hnswEfConfig(ef) hnswEfConfig.put(ef, (oldTime + timeTaken, oldRecall + recall)) } annoyConfigCopy.keys.foreach { nodes => val (nn, timeTaken) = time( Await.result( annoyQuery .query(embedding, neighbours, AnnoyRuntimeParams(nodesToExplore = Some(nodes))) .map(_.toSet))) val recall = (list.intersect(nn).size) * 1.0f / neighbours val (oldTime, oldRecall) = annoyConfig(nodes) annoyConfig.put(nodes, (oldTime + timeTaken, oldRecall + recall)) } } println( s"Bruteforce avg query time : ${bruteForceTime / testDataSetSize} ms") efSearch.foreach { ef => val data = hnswEfConfig(ef) println( s"Hnsw avg recall and time with query ef : $ef => ${data._2 / testDataSetSize} ${data._1 / testDataSetSize} ms" ) } nodesToExplore.foreach { n => val data = annoyConfig(n) println( s"Annoy avg recall and time with nodes_to_explore : $n => ${data._2 / testDataSetSize} ${data._1 / testDataSetSize} ms" ) } exec.shutdown() } def time[T](fn: => T): (T, Long) = { val start = System.currentTimeMillis() val result = fn val end = System.currentTimeMillis() (result, (end - start)) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/faiss/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/mapdb", "ann/src/main/java/com/twitter/ann/faiss", "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/scala/com/twitter/ann/serialization", "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", "mediaservices/commons/src/main/scala:futuretracker", "src/java/com/twitter/common_internal/hadoop", "src/java/com/twitter/search/common/file", "src/scala/com/twitter/ml/api/embedding", ], exports = [ "ann/src/main/scala/com/twitter/ann/common", "src/java/com/twitter/common_internal/hadoop", "src/java/com/twitter/search/common/file", "src/scala/com/twitter/ml/api/embedding", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/faiss/FaissCommon.scala ================================================ package com.twitter.ann.faiss import com.twitter.ann.common.thriftscala.FaissRuntimeParam import com.twitter.bijection.Injection import scala.util.Failure import scala.util.Success import scala.util.Try import com.twitter.ann.common.thriftscala.{RuntimeParams => ServiceRuntimeParams} import com.twitter.search.common.file.AbstractFile object FaissCommon { val RuntimeParamsInjection: Injection[FaissParams, ServiceRuntimeParams] = new Injection[FaissParams, ServiceRuntimeParams] { override def apply(scalaParams: FaissParams): ServiceRuntimeParams = { ServiceRuntimeParams.FaissParam( FaissRuntimeParam( scalaParams.nprobe, scalaParams.quantizerEf, scalaParams.quantizerKFactorRF, scalaParams.quantizerNprobe, scalaParams.ht) ) } override def invert(thriftParams: ServiceRuntimeParams): Try[FaissParams] = thriftParams match { case ServiceRuntimeParams.FaissParam(faissParam) => Success( FaissParams( faissParam.nprobe, faissParam.quantizerEf, faissParam.quantizerKfactorRf, faissParam.quantizerNprobe, faissParam.ht)) case p => Failure(new IllegalArgumentException(s"Expected FaissParams got $p")) } } def isValidFaissIndex(path: AbstractFile): Boolean = { path.isDirectory && path.hasSuccessFile && path.getChild("faiss.index").exists() } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/faiss/FaissIndex.scala ================================================ package com.twitter.ann.faiss import com.twitter.ann.common.Queryable import com.twitter.ann.common._ import com.twitter.search.common.file.AbstractFile import com.twitter.util.logging.Logging case class FaissParams( nprobe: Option[Int], quantizerEf: Option[Int], quantizerKFactorRF: Option[Int], quantizerNprobe: Option[Int], ht: Option[Int]) extends RuntimeParams { override def toString: String = s"FaissParams(${toLibraryString})" def toLibraryString: String = Seq( nprobe.map { n => s"nprobe=${n}" }, quantizerEf.map { ef => s"quantizer_efSearch=${ef}" }, quantizerKFactorRF.map { k => s"quantizer_k_factor_rf=${k}" }, quantizerNprobe.map { n => s"quantizer_nprobe=${n}" }, ht.map { ht => s"ht=${ht}" }, ).flatten.mkString(",") } object FaissIndex { def loadIndex[T, D <: Distance[D]]( outerDimension: Int, outerMetric: Metric[D], directory: AbstractFile ): Queryable[T, FaissParams, D] = { new QueryableIndexAdapter[T, D] with Logging { protected val metric: Metric[D] = outerMetric protected val dimension: Int = outerDimension protected val index: Index = { info(s"Loading faiss with ${swigfaiss.get_compile_options()}") QueryableIndexAdapter.loadJavaIndex(directory) } } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/faiss/FaissIndexer.scala ================================================ package com.twitter.ann.faiss import com.google.common.base.Preconditions import com.twitter.ann.common.Cosine import com.twitter.ann.common.Distance import com.twitter.ann.common.EntityEmbedding import com.twitter.ann.common.IndexOutputFile import com.twitter.ann.common.InnerProduct import com.twitter.ann.common.L2 import com.twitter.ann.common.Metric import com.twitter.ml.api.embedding.EmbeddingMath import com.twitter.scalding.Execution import com.twitter.scalding.TypedPipe import com.twitter.search.common.file.AbstractFile import com.twitter.search.common.file.FileUtils import com.twitter.util.logging.Logging import java.io.File import scala.util.Random trait FaissIndexer extends Logging { /** * Produce faiss index file specified by factory string * * @param pipe Embeddings to be indexed * @param sampleRate Fraction of embeddings used for training. Regardless of this parameter, all embeddings are present in the output. * @param factoryString Faiss factory string, see https://github.com/facebookresearch/faiss/wiki/The-index-factory * @param metric Metric to use * @param outputDirectory Directory where _SUCCESS and faiss.index will be written. */ def build[D <: Distance[D]]( pipe: TypedPipe[EntityEmbedding[Long]], sampleRate: Float, factoryString: String, metric: Metric[D], outputDirectory: AbstractFile ): Execution[Unit] = { outputDirectory.mkdirs() Preconditions.checkState( outputDirectory.canRead, "Failed to create parent directories for %s", outputDirectory.toString) val maybeNormalizedPipe = if (l2Normalize(metric)) { pipe.map { idAndEmbedding => EntityEmbedding(idAndEmbedding.id, EmbeddingMath.Float.normalize(idAndEmbedding.embedding)) } } else { pipe } maybeNormalizedPipe.toIterableExecution.flatMap { annEmbeddings => logger.info(s"${factoryString}") val t1 = System.nanoTime buildAndWriteFaissIndex( Random.shuffle(annEmbeddings), sampleRate, factoryString, metric, new IndexOutputFile(outputDirectory)) val duration = (System.nanoTime - t1) / 1e9d logger.info(s"It took ${duration}s to build and index") Execution.unit } } def buildAndWriteFaissIndex[D <: Distance[D]]( entities: Iterable[EntityEmbedding[Long]], sampleRate: Float, factoryString: String, metricType: Metric[D], outputDirectory: IndexOutputFile ): Unit = { val metric = parseMetric(metricType) val datasetSize = entities.size.toLong val dimensions = entities.head.embedding.length logger.info(s"There are $datasetSize embeddings") logger.info(s"Faiss compile options are ${swigfaiss.get_compile_options()}") logger.info(s"OMP threads count is ${swigfaiss.omp_get_max_threads()}") val index = swigfaiss.index_factory(dimensions, factoryString, metric) index.setVerbose(true) val idMap = new IndexIDMap(index) val trainingSetSize = Math.min(datasetSize, Math.round(datasetSize * sampleRate)) val ids = toIndexVector(entities) val fullDataset = toFloatVector(dimensions, entities) logger.info("Finished bridging full dataset") idMap.train(trainingSetSize, fullDataset.data()) logger.info("Finished training") idMap.add_with_ids(datasetSize, fullDataset.data(), ids) logger.info("Added data to the index") val tmpFile = File.createTempFile("faiss.index", ".tmp") swigfaiss.write_index(idMap, tmpFile.toString) logger.info(s"Wrote to tmp file ${tmpFile.toString}") copyToOutputAndCreateSuccess(FileUtils.getFileHandle(tmpFile.toString), outputDirectory) logger.info("Copied file") } private def copyToOutputAndCreateSuccess( tmpFile: AbstractFile, outputDirectory: IndexOutputFile ) = { val outputFile = outputDirectory.createFile("faiss.index") logger.info(s"Final output file is ${outputFile.getPath()}") outputFile.copyFrom(tmpFile.getByteSource.openStream()) outputDirectory.createSuccessFile() } private def toFloatVector( dimensions: Int, entities: Iterable[EntityEmbedding[Long]] ): FloatVector = { require(entities.nonEmpty) val vector = new FloatVector() vector.reserve(dimensions.toLong * entities.size.toLong) for (entity <- entities) { for (value <- entity.embedding) { vector.push_back(value) } } vector } private def toIndexVector(embeddings: Iterable[EntityEmbedding[Long]]): LongVector = { require(embeddings.nonEmpty) val vector = new LongVector() vector.reserve(embeddings.size) for (embedding <- embeddings) { vector.push_back(embedding.id) } vector } private def parseMetric[D <: Distance[D]](metric: Metric[D]): MetricType = metric match { case L2 => MetricType.METRIC_L2 case InnerProduct => MetricType.METRIC_INNER_PRODUCT case Cosine => MetricType.METRIC_INNER_PRODUCT case _ => throw new AbstractMethodError(s"Not implemented for metric ${metric}") } private def l2Normalize[D <: Distance[D]](metric: Metric[D]): Boolean = metric match { case Cosine => true case _ => false } } object FaissIndexer extends FaissIndexer {} ================================================ FILE: ann/src/main/scala/com/twitter/ann/faiss/HourlyDirectoryWithSuccessFileListing.scala ================================================ package com.twitter.ann.faiss import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.search.common.file.AbstractFile import com.twitter.search.common.file.FileUtils import com.twitter.util.Return import com.twitter.util.Throw import com.twitter.util.Time import com.twitter.util.Try import com.twitter.util.logging.Logging import java.util.Locale object HourlyDirectoryWithSuccessFileListing extends Logging { private val SUCCESS_FILE_NAME = "_SUCCESS" def listHourlyIndexDirectories( root: AbstractFile, startingFrom: Time, count: Int, lookbackInterval: Int ): Seq[AbstractFile] = listingStep(root, startingFrom, count, lookbackInterval) private def listingStep( root: AbstractFile, startingFrom: Time, remainingDirectoriesToFind: Int, remainingAttempts: Int ): List[AbstractFile] = { if (remainingDirectoriesToFind == 0 || remainingAttempts == 0) { return List.empty } val head = getSuccessfulDirectoryForDate(root, startingFrom) val previousHour = startingFrom - 1.hour head match { case Throw(e) => listingStep(root, previousHour, remainingDirectoriesToFind, remainingAttempts - 1) case Return(directory) => directory :: listingStep(root, previousHour, remainingDirectoriesToFind - 1, remainingAttempts - 1) } } private def getSuccessfulDirectoryForDate( root: AbstractFile, date: Time ): Try[AbstractFile] = { val folder = root.getPath + "/" + date.format("yyyy/MM/dd/HH", Locale.ROOT) val successPath = folder + "/" + SUCCESS_FILE_NAME debug(s"Checking ${successPath}") Try(FileUtils.getFileHandle(successPath)).flatMap { file => if (file.canRead) { Try(FileUtils.getFileHandle(folder)) } else { Throw(new IllegalArgumentException(s"Found ${file.toString} but can't read it")) } } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/faiss/HourlyShardedIndex.scala ================================================ package com.twitter.ann.faiss import com.twitter.ann.common.Distance import com.twitter.ann.common.MemoizedInEpochs import com.twitter.ann.common.Metric import com.twitter.ann.common.Task import com.twitter.finagle.stats.StatsReceiver import com.twitter.search.common.file.AbstractFile import com.twitter.util.Duration import com.twitter.util.Future import com.twitter.util.Time import com.twitter.util.Try import com.twitter.util.logging.Logging import java.util.concurrent.atomic.AtomicReference object HourlyShardedIndex { def loadIndex[T, D <: Distance[D]]( dimension: Int, metric: Metric[D], directory: AbstractFile, shardsToLoad: Int, shardWatchInterval: Duration, lookbackInterval: Int, statsReceiver: StatsReceiver ): HourlyShardedIndex[T, D] = { new HourlyShardedIndex[T, D]( metric, dimension, directory, shardsToLoad, shardWatchInterval, lookbackInterval, statsReceiver) } } class HourlyShardedIndex[T, D <: Distance[D]]( outerMetric: Metric[D], outerDimension: Int, directory: AbstractFile, shardsToLoad: Int, shardWatchInterval: Duration, lookbackInterval: Int, override protected val statsReceiver: StatsReceiver) extends QueryableIndexAdapter[T, D] with Logging with Task { // QueryableIndexAdapter protected val metric: Metric[D] = outerMetric protected val dimension: Int = outerDimension protected def index: Index = { castedIndex.get() } // Task trait protected def task(): Future[Unit] = Future.value(reloadShards()) protected def taskInterval: Duration = shardWatchInterval private def loadIndex(directory: AbstractFile): Try[Index] = Try(QueryableIndexAdapter.loadJavaIndex(directory)) private val shardsCache = new MemoizedInEpochs[AbstractFile, Index](loadIndex) // Destroying original index invalidate casted index. Keep a reference to both. private val originalIndex = new AtomicReference[IndexShards]() private val castedIndex = new AtomicReference[Index]() private def reloadShards(): Unit = { val freshDirectories = HourlyDirectoryWithSuccessFileListing.listHourlyIndexDirectories( directory, Time.now, shardsToLoad, lookbackInterval) if (shardsCache.currentEpochKeys == freshDirectories.toSet) { info("Not reloading shards, as they're exactly same") } else { val shards = shardsCache.epoch(freshDirectories) val indexShards = new IndexShards(dimension, false, false) for (shard <- shards) { indexShards.add_shard(shard) } replaceIndex(() => { castedIndex.set(swigfaiss.upcast_IndexShards(indexShards)) originalIndex.set(indexShards) }) // Potentially it's time to drop huge native index from memory, ask for GC System.gc() } require(castedIndex.get() != null, "Failed to find any shards during startup") } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/faiss/QueryableIndexAdapter.scala ================================================ package com.twitter.ann.faiss import com.twitter.ann.common.Cosine import com.twitter.ann.common.Distance import com.twitter.ann.common.EmbeddingType.EmbeddingVector import com.twitter.ann.common.Metric import com.twitter.ann.common.NeighborWithDistance import com.twitter.ann.common.Queryable import com.twitter.ml.api.embedding.EmbeddingMath import com.twitter.search.common.file.AbstractFile import com.twitter.search.common.file.FileUtils import com.twitter.util.Future import com.twitter.util.logging.Logging import java.io.File import java.util.concurrent.locks.ReentrantReadWriteLock object QueryableIndexAdapter extends Logging { // swigfaiss.read_index doesn't support hdfs files, hence a copy to temporary directory def loadJavaIndex(directory: AbstractFile): Index = { val indexFile = directory.getChild("faiss.index") val tmpFile = File.createTempFile("faiss.index", ".tmp") val tmpAbstractFile = FileUtils.getFileHandle(tmpFile.toString) indexFile.copyTo(tmpAbstractFile) val index = swigfaiss.read_index(tmpAbstractFile.getPath) if (!tmpFile.delete()) { error(s"Failed to delete ${tmpFile.toString}") } index } } trait QueryableIndexAdapter[T, D <: Distance[D]] extends Queryable[T, FaissParams, D] { this: Logging => private val MAX_COSINE_DISTANCE = 1f protected def index: Index protected val metric: Metric[D] protected val dimension: Int private def maybeNormalizeEmbedding(embeddingVector: EmbeddingVector): EmbeddingVector = { // There is no direct support for Cosine, but l2norm + ip == Cosine by definition if (metric == Cosine) { EmbeddingMath.Float.normalize(embeddingVector) } else { embeddingVector } } private def maybeTranslateToCosineDistanceInplace(array: floatArray, len: Int): Unit = { // Faiss reports Cosine similarity while we need Cosine distance. if (metric == Cosine) { for (index <- 0 until len) { val similarity = array.getitem(index) if (similarity < 0 || similarity > 1) { warn(s"Expected similarity to be between 0 and 1, got ${similarity} instead") array.setitem(index, MAX_COSINE_DISTANCE) } else { array.setitem(index, 1 - similarity) } } } } private val paramsLock = new ReentrantReadWriteLock() private var currentParams: Option[String] = None // Assume that parameters rarely change and try read lock first private def ensuringParams[R](parameterString: String, f: () => R): R = { paramsLock.readLock().lock() try { if (currentParams.contains(parameterString)) { return f() } } finally { paramsLock.readLock().unlock() } paramsLock.writeLock().lock() try { currentParams = Some(parameterString) new ParameterSpace().set_index_parameters(index, parameterString) f() } finally { paramsLock.writeLock().unlock() } } def replaceIndex(f: () => Unit): Unit = { paramsLock.writeLock().lock() try { currentParams = None f() } finally { paramsLock.writeLock().unlock() } } def query( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: FaissParams ): Future[List[T]] = { Future.value( ensuringParams( runtimeParams.toLibraryString, () => { val distances = new floatArray(numOfNeighbors) val indexes = new LongVector() indexes.resize(numOfNeighbors) val normalizedEmbedding = maybeNormalizeEmbedding(embedding) index.search( // Number of query embeddings 1, // Array of query embeddings toFloatArray(normalizedEmbedding).cast(), // Number of neighbours to return numOfNeighbors, // Location to store neighbour distances distances.cast(), // Location to store neighbour identifiers indexes ) // This is a shortcoming of current swig bindings // Nothing prevents JVM from freeing distances while inside index.search // This might be removed once we start passing FloatVector // Why java.lang.ref.Reference.reachabilityFence doesn't compile? debug(distances) toSeq(indexes, numOfNeighbors).toList.asInstanceOf[List[T]] } )) } def queryWithDistance( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: FaissParams ): Future[List[NeighborWithDistance[T, D]]] = { Future.value( ensuringParams( runtimeParams.toLibraryString, () => { val distances = new floatArray(numOfNeighbors) val indexes = new LongVector() indexes.resize(numOfNeighbors) val normalizedEmbedding = maybeNormalizeEmbedding(embedding) index.search( // Number of query embeddings 1, // Array of query embeddings toFloatArray(normalizedEmbedding).cast(), // Number of neighbours to return numOfNeighbors, // Location to store neighbour distances distances.cast(), // Location to store neighbour identifiers indexes ) val ids = toSeq(indexes, numOfNeighbors).toList.asInstanceOf[List[T]] maybeTranslateToCosineDistanceInplace(distances, numOfNeighbors) val distancesSeq = toSeq(distances, numOfNeighbors) ids.zip(distancesSeq).map { case (id, distance) => NeighborWithDistance(id, metric.fromAbsoluteDistance(distance)) } } )) } private def toFloatArray(emb: EmbeddingVector): floatArray = { val nativeArray = new floatArray(emb.length) for ((value, aIdx) <- emb.iterator.zipWithIndex) { nativeArray.setitem(aIdx, value) } nativeArray } private def toSeq(vector: LongVector, len: Long): Seq[Long] = { (0L until len).map(vector.at) } private def toSeq(array: floatArray, len: Int): Seq[Float] = { (0 until len).map(array.getitem) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/featurestore/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], tags = ["bazel-compatible"], dependencies = [ "ann/src/main/scala/com/twitter/ann/common", "src/scala/com/twitter/ml/featurestore/lib", "src/scala/com/twitter/ml/featurestore/lib/online", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/featurestore/FeatureStoreEmbeddingProducer.scala ================================================ package com.twitter.ann.featurestore import com.twitter.ann.common.EmbeddingProducer import com.twitter.finagle.stats.{InMemoryStatsReceiver, StatsReceiver} import com.twitter.ml.api.embedding.{Embedding, EmbeddingSerDe} import com.twitter.ml.api.thriftscala import com.twitter.ml.api.thriftscala.{Embedding => TEmbedding} import com.twitter.ml.featurestore.lib.dataset.online.VersionedOnlineAccessDataset import com.twitter.ml.featurestore.lib.{EntityId, RawFloatTensor} import com.twitter.ml.featurestore.lib.dataset.DatasetParams import com.twitter.ml.featurestore.lib.entity.EntityWithId import com.twitter.ml.featurestore.lib.feature.{BoundFeature, BoundFeatureSet} import com.twitter.ml.featurestore.lib.online.{FeatureStoreClient, FeatureStoreRequest} import com.twitter.ml.featurestore.lib.params.FeatureStoreParams import com.twitter.stitch.Stitch import com.twitter.strato.opcontext.Attribution import com.twitter.strato.client.Client object FeatureStoreEmbeddingProducer { def apply[T <: EntityId]( dataset: VersionedOnlineAccessDataset[T, TEmbedding], version: Long, boundFeature: BoundFeature[T, RawFloatTensor], client: Client, statsReceiver: StatsReceiver = new InMemoryStatsReceiver, featureStoreAttributions: Seq[Attribution] = Seq.empty ): EmbeddingProducer[EntityWithId[T]] = { val featureStoreParams = FeatureStoreParams( perDataset = Map( dataset.id -> DatasetParams(datasetVersion = Some(version)) ), global = DatasetParams(attributions = featureStoreAttributions) ) val featureStoreClient = FeatureStoreClient( BoundFeatureSet(boundFeature), client, statsReceiver, featureStoreParams ) new FeatureStoreEmbeddingProducer(boundFeature, featureStoreClient) } } private[featurestore] class FeatureStoreEmbeddingProducer[T <: EntityId]( boundFeature: BoundFeature[T, RawFloatTensor], featureStoreClient: FeatureStoreClient) extends EmbeddingProducer[EntityWithId[T]] { // Looks up embedding from online feature store for an entity. override def produceEmbedding(input: EntityWithId[T]): Stitch[Option[Embedding[Float]]] = { val featureStoreRequest = FeatureStoreRequest( entityIds = Seq(input) ) Stitch.callFuture(featureStoreClient(featureStoreRequest).map { predictionRecord => predictionRecord.getFeatureValue(boundFeature) match { case Some(featureValue) => { val embedding = EmbeddingSerDe.floatEmbeddingSerDe.fromThrift( thriftscala.Embedding(Some(featureValue.value)) ) Some(embedding) } case _ => None } }) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/file_store/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/storehaus:core", "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", "mediaservices/commons/src/main/scala", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/file_store/ReadableIndexIdFileStore.scala ================================================ package com.twitter.ann.file_store import com.twitter.ann.common.thriftscala.FileBasedIndexIdStore import com.twitter.bijection.Injection import com.twitter.mediaservices.commons.codec.{ArrayByteBufferCodec, ThriftByteBufferCodec} import com.twitter.search.common.file.AbstractFile import com.twitter.storehaus.ReadableStore import java.nio.ByteBuffer object ReadableIndexIdFileStore { /** * @param file : File path to read serialized long indexId <-> Id mapping from. * @param injection: Injection to convert bytes to Id. * @tparam V: Type of Id * @return File based Readable Store */ def apply[V]( file: AbstractFile, injection: Injection[V, Array[Byte]] ): ReadableStore[Long, V] = { val codec = new ThriftByteBufferCodec(FileBasedIndexIdStore) val store: Map[Long, V] = codec .decode(loadFile(file)) .indexIdMap .getOrElse(Map.empty[Long, ByteBuffer]) .toMap .mapValues(value => injection.invert(ArrayByteBufferCodec.decode(value)).get) ReadableStore.fromMap[Long, V](store) } private[this] def loadFile(file: AbstractFile): ByteBuffer = { ArrayByteBufferCodec.encode(file.getByteSource.read()) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/file_store/WritableIndexIdFileStore.scala ================================================ package com.twitter.ann.file_store import com.twitter.ann.common.IndexOutputFile import com.twitter.ann.common.thriftscala.FileBasedIndexIdStore import com.twitter.bijection.Injection import com.twitter.mediaservices.commons.codec.ArrayByteBufferCodec import com.twitter.mediaservices.commons.codec.ThriftByteBufferCodec import com.twitter.storehaus.Store import com.twitter.util.Future import java.util.concurrent.{ConcurrentHashMap => JConcurrentHashMap} import scala.collection.JavaConverters._ object WritableIndexIdFileStore { /** * @param injection: Injection to convert typed Id to bytes. * @tparam V: Type of Id * @return File based Writable Store */ def apply[V]( injection: Injection[V, Array[Byte]] ): WritableIndexIdFileStore[V] = { new WritableIndexIdFileStore[V]( new JConcurrentHashMap[Long, Option[V]], injection ) } } class WritableIndexIdFileStore[V] private ( map: JConcurrentHashMap[Long, Option[V]], injection: Injection[V, Array[Byte]]) extends Store[Long, V] { private[this] val store = Store.fromJMap(map) override def get(k: Long): Future[Option[V]] = { store.get(k) } override def put(kv: (Long, Option[V])): Future[Unit] = { store.put(kv) } /** * Serialize and store the mapping in thrift format * @param file : File path to store serialized long indexId <-> Id mapping */ def save(file: IndexOutputFile): Unit = { saveThrift(toThrift(), file) } def getInjection: Injection[V, Array[Byte]] = injection private[this] def toThrift(): FileBasedIndexIdStore = { val indexIdMap = map.asScala .collect { case (key, Some(value)) => (key, ArrayByteBufferCodec.encode(injection.apply(value))) } FileBasedIndexIdStore(Some(indexIdMap)) } private[this] def saveThrift(thriftObj: FileBasedIndexIdStore, file: IndexOutputFile): Unit = { val codec = new ThriftByteBufferCodec(FileBasedIndexIdStore) val bytes = ArrayByteBufferCodec.decode(codec.encode(thriftObj)) val outputStream = file.getOutputStream() outputStream.write(bytes) outputStream.close() } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/hnsw/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/mapdb", "ann/src/main/java/com/twitter/ann/hnsw", "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/scala/com/twitter/ann/serialization", "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", "mediaservices/commons/src/main/scala:futuretracker", "src/java/com/twitter/common_internal/hadoop", "src/java/com/twitter/search/common/file", "src/scala/com/twitter/ml/api/embedding", ], exports = [ "ann/src/main/scala/com/twitter/ann/common", "src/java/com/twitter/common_internal/hadoop", "src/java/com/twitter/search/common/file", "src/scala/com/twitter/ml/api/embedding", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/hnsw/DistanceFunctionGenerator.scala ================================================ package com.twitter.ann.hnsw import com.twitter.ann.common.EmbeddingType.EmbeddingVector import com.twitter.ann.common.{Cosine, Distance, InnerProduct, Metric} private[hnsw] object DistanceFunctionGenerator { def apply[T, D <: Distance[D]]( metric: Metric[D], idToEmbeddingFn: (T) => EmbeddingVector ): DistanceFunctionGenerator[T] = { // Use InnerProduct for cosine and normalize the vectors before appending and querying. val updatedMetric = metric match { case Cosine => InnerProduct case _ => metric } val distFnIndex = new DistanceFunction[T, T] { override def distance(id1: T, id2: T) = updatedMetric.absoluteDistance( idToEmbeddingFn(id1), idToEmbeddingFn(id2) ) } val distFnQuery = new DistanceFunction[EmbeddingVector, T] { override def distance(embedding: EmbeddingVector, id: T) = updatedMetric.absoluteDistance(embedding, idToEmbeddingFn(id)) } DistanceFunctionGenerator(distFnIndex, distFnQuery, metric == Cosine) } } private[hnsw] case class DistanceFunctionGenerator[T]( index: DistanceFunction[T, T], query: DistanceFunction[EmbeddingVector, T], shouldNormalize: Boolean) ================================================ FILE: ann/src/main/scala/com/twitter/ann/hnsw/Hnsw.scala ================================================ package com.twitter.ann.hnsw import com.google.common.annotations.VisibleForTesting import com.twitter.ann.common.EmbeddingType._ import com.twitter.ann.common.Metric.toThrift import com.twitter.ann.common._ import com.twitter.ann.common.thriftscala.DistanceMetric import com.twitter.ann.hnsw.HnswIndex.RandomProvider import com.twitter.util.Future import java.util.Random import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ThreadLocalRandom import java.util.concurrent.locks.Lock import java.util.concurrent.locks.ReentrantLock import scala.collection.JavaConverters._ private[hnsw] object Hnsw { private[hnsw] def apply[T, D <: Distance[D]]( dimension: Int, metric: Metric[D], efConstruction: Int, maxM: Int, expectedElements: Int, futurePool: ReadWriteFuturePool, idEmbeddingMap: IdEmbeddingMap[T] ): Hnsw[T, D] = { val randomProvider = new RandomProvider { override def get(): Random = ThreadLocalRandom.current() } val distFn = DistanceFunctionGenerator(metric, (key: T) => idEmbeddingMap.get(key)) val internalIndex = new HnswIndex[T, EmbeddingVector]( distFn.index, distFn.query, efConstruction, maxM, expectedElements, randomProvider ) new Hnsw[T, D]( dimension, metric, internalIndex, futurePool, idEmbeddingMap, distFn.shouldNormalize, LockedAccess.apply(expectedElements) ) } } private[hnsw] object LockedAccess { protected[hnsw] def apply[T](expectedElements: Int): LockedAccess[T] = DefaultLockedAccess(new ConcurrentHashMap[T, Lock](expectedElements)) protected[hnsw] def apply[T](): LockedAccess[T] = DefaultLockedAccess(new ConcurrentHashMap[T, Lock]()) } private[hnsw] case class DefaultLockedAccess[T](locks: ConcurrentHashMap[T, Lock]) extends LockedAccess[T] { override def lockProvider(item: T) = locks.computeIfAbsent(item, (_: T) => new ReentrantLock()) } private[hnsw] trait LockedAccess[T] { protected def lockProvider(item: T): Lock def lock[K](item: T)(fn: => K): K = { val lock = lockProvider(item) lock.lock() try { fn } finally { lock.unlock() } } } @VisibleForTesting private[hnsw] class Hnsw[T, D <: Distance[D]]( dimension: Int, metric: Metric[D], hnswIndex: HnswIndex[T, EmbeddingVector], readWriteFuturePool: ReadWriteFuturePool, idEmbeddingMap: IdEmbeddingMap[T], shouldNormalize: Boolean, lockedAccess: LockedAccess[T] = LockedAccess.apply[T]()) extends Appendable[T, HnswParams, D] with Queryable[T, HnswParams, D] with Updatable[T] { override def append(entity: EntityEmbedding[T]): Future[Unit] = { readWriteFuturePool.write { val indexDimension = entity.embedding.length assert( toThrift(metric) == DistanceMetric.EditDistance || indexDimension == dimension, s"Dimension mismatch for index(${indexDimension}) and embedding($dimension)" ) lockedAccess.lock(entity.id) { // To make this thread-safe, we are using ConcurrentHashMap#putIfAbsent underneath, // so if there is a pre-existing item, put() will return something that is not null val embedding = idEmbeddingMap.putIfAbsent(entity.id, updatedEmbedding(entity.embedding)) if (embedding == null) { // New element - insert into the index hnswIndex.insert(entity.id) } else { // Existing element - update the embedding and graph structure throw new IllegalDuplicateInsertException( "Append method does not permit duplicates (try using update method): " + entity.id) } } } onFailure { e => Future.exception(e) } } override def toQueryable: Queryable[T, HnswParams, D] = this override def query( embedding: EmbeddingVector, numOfNeighbours: Int, runtimeParams: HnswParams ): Future[List[T]] = { queryWithDistance(embedding, numOfNeighbours, runtimeParams) .map(_.map(_.neighbor)) } override def queryWithDistance( embedding: EmbeddingVector, numOfNeighbours: Int, runtimeParams: HnswParams ): Future[List[NeighborWithDistance[T, D]]] = { val indexDimension = embedding.length assert( toThrift(metric) == DistanceMetric.EditDistance || indexDimension == dimension, s"Dimension mismatch for index(${indexDimension}) and embedding($dimension)" ) readWriteFuturePool.read { hnswIndex .searchKnn(updatedEmbedding(embedding), numOfNeighbours, runtimeParams.ef) .asScala .map { nn => NeighborWithDistance( nn.getItem, metric.fromAbsoluteDistance(nn.getDistance) ) } .toList } } private[this] def updatedEmbedding(embedding: EmbeddingVector): EmbeddingVector = { if (shouldNormalize) { MetricUtil.norm(embedding) } else { embedding } } def getIndex: HnswIndex[T, EmbeddingVector] = hnswIndex def getDimen: Int = dimension def getMetric: Metric[D] = metric def getIdEmbeddingMap: IdEmbeddingMap[T] = idEmbeddingMap override def update( entity: EntityEmbedding[T] ): Future[Unit] = { readWriteFuturePool.write { val indexDimension = entity.embedding.length assert( toThrift(metric) == DistanceMetric.EditDistance || indexDimension == dimension, s"Dimension mismatch for index(${indexDimension}) and embedding($dimension)" ) lockedAccess.lock(entity.id) { val embedding = idEmbeddingMap.put(entity.id, updatedEmbedding(entity.embedding)) if (embedding == null) { // New element - insert into the index hnswIndex.insert(entity.id) } else { // Existing element - update the embedding and graph structure hnswIndex.reInsert(entity.id); } } } onFailure { e => Future.exception(e) } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/hnsw/HnswCommon.scala ================================================ package com.twitter.ann.hnsw import com.twitter.ann.common.RuntimeParams import com.twitter.ann.common.thriftscala.HnswIndexMetadata import com.twitter.ann.common.thriftscala.HnswRuntimeParam import com.twitter.ann.common.thriftscala.{RuntimeParams => ServiceRuntimeParams} import com.twitter.bijection.Injection import com.twitter.mediaservices.commons.codec.ThriftByteBufferCodec import com.twitter.search.common.file.AbstractFile import scala.util.Failure import scala.util.Success import scala.util.Try object HnswCommon { private[hnsw] lazy val MetadataCodec = new ThriftByteBufferCodec(HnswIndexMetadata) private[hnsw] val MetaDataFileName = "hnsw_index_metadata" private[hnsw] val EmbeddingMappingFileName = "hnsw_embedding_mapping" private[hnsw] val InternalIndexDir = "hnsw_internal_index" private[hnsw] val HnswInternalMetadataFileName = "hnsw_internal_metadata" private[hnsw] val HnswInternalGraphFileName = "hnsw_internal_graph" val RuntimeParamsInjection: Injection[HnswParams, ServiceRuntimeParams] = new Injection[HnswParams, ServiceRuntimeParams] { override def apply(scalaParams: HnswParams): ServiceRuntimeParams = { ServiceRuntimeParams.HnswParam( HnswRuntimeParam( scalaParams.ef ) ) } override def invert(thriftParams: ServiceRuntimeParams): Try[HnswParams] = thriftParams match { case ServiceRuntimeParams.HnswParam(hnswParam) => Success( HnswParams(hnswParam.ef) ) case p => Failure(new IllegalArgumentException(s"Expected HnswRuntimeParam got $p")) } } def isValidHnswIndex(path: AbstractFile): Boolean = { path.isDirectory && path.hasSuccessFile && path.getChild(MetaDataFileName).exists() && path.getChild(EmbeddingMappingFileName).exists() && path.getChild(InternalIndexDir).exists() && path.getChild(InternalIndexDir).getChild(HnswInternalMetadataFileName).exists() && path.getChild(InternalIndexDir).getChild(HnswInternalGraphFileName).exists() } } /** * Hnsw runtime params * @param ef: The size of the dynamic list for the nearest neighbors (used during the search). * Higher ef leads to more accurate but slower search. * ef cannot be set lower than the number of queried nearest neighbors k. * The value ef of can be anything between k and the size of the dataset. */ case class HnswParams(ef: Int) extends RuntimeParams { override def toString: String = s"HnswParams(ef = $ef)" } ================================================ FILE: ann/src/main/scala/com/twitter/ann/hnsw/HnswIOUtil.scala ================================================ package com.twitter.ann.hnsw import com.google.common.annotations.VisibleForTesting import com.twitter.ann.common.EmbeddingType.EmbeddingVector import com.twitter.ann.common.thriftscala.HnswIndexMetadata import com.twitter.ann.common.Distance import com.twitter.ann.common.EntityEmbedding import com.twitter.ann.common.Metric import com.twitter.ann.hnsw.HnswCommon._ import com.twitter.ann.serialization.PersistedEmbeddingInjection import com.twitter.ann.serialization.ThriftIteratorIO import com.twitter.ann.serialization.thriftscala.PersistedEmbedding import com.twitter.bijection.Injection import com.twitter.mediaservices.commons.codec.ArrayByteBufferCodec import com.twitter.search.common.file.AbstractFile import java.io.BufferedInputStream import java.io.BufferedOutputStream import java.io.OutputStream private[hnsw] object HnswIOUtil { private val BufferSize = 64 * 1024 // Default 64Kb @VisibleForTesting private[hnsw] def loadEmbeddings[T]( embeddingFile: AbstractFile, injection: Injection[T, Array[Byte]], idEmbeddingMap: IdEmbeddingMap[T], ): IdEmbeddingMap[T] = { val inputStream = { val stream = embeddingFile.getByteSource.openStream() if (stream.isInstanceOf[BufferedInputStream]) { stream } else { new BufferedInputStream(stream, BufferSize) } } val thriftIteratorIO = new ThriftIteratorIO[PersistedEmbedding](PersistedEmbedding) val iterator = thriftIteratorIO.fromInputStream(inputStream) val embeddingInjection = new PersistedEmbeddingInjection(injection) try { iterator.foreach { persistedEmbedding => val embedding = embeddingInjection.invert(persistedEmbedding).get idEmbeddingMap.putIfAbsent(embedding.id, embedding.embedding) Unit } } finally { inputStream.close() } idEmbeddingMap } @VisibleForTesting private[hnsw] def saveEmbeddings[T]( stream: OutputStream, injection: Injection[T, Array[Byte]], iter: Iterator[(T, EmbeddingVector)] ): Unit = { val thriftIteratorIO = new ThriftIteratorIO[PersistedEmbedding](PersistedEmbedding) val embeddingInjection = new PersistedEmbeddingInjection(injection) val iterator = iter.map { case (id, emb) => embeddingInjection(EntityEmbedding(id, emb)) } val outputStream = { if (stream.isInstanceOf[BufferedOutputStream]) { stream } else { new BufferedOutputStream(stream, BufferSize) } } try { thriftIteratorIO.toOutputStream(iterator, outputStream) } finally { outputStream.close() } } @VisibleForTesting private[hnsw] def saveIndexMetadata( dimension: Int, metric: Metric[_ <: Distance[_]], numElements: Int, metadataStream: OutputStream ): Unit = { val metadata = HnswIndexMetadata( dimension, Metric.toThrift(metric), numElements ) val bytes = ArrayByteBufferCodec.decode(MetadataCodec.encode(metadata)) metadataStream.write(bytes) metadataStream.close() } @VisibleForTesting private[hnsw] def loadIndexMetadata( metadataFile: AbstractFile ): HnswIndexMetadata = { MetadataCodec.decode( ArrayByteBufferCodec.encode(metadataFile.getByteSource.read()) ) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/hnsw/IdEmbeddingMap.scala ================================================ package com.twitter.ann.hnsw import com.twitter.ann.common.EmbeddingType._ import java.io.OutputStream trait IdEmbeddingMap[T] { def putIfAbsent(id: T, embedding: EmbeddingVector): EmbeddingVector def put(id: T, embedding: EmbeddingVector): EmbeddingVector def get(id: T): EmbeddingVector def iter(): Iterator[(T, EmbeddingVector)] def size(): Int def toDirectory(embeddingFileOutputStream: OutputStream): Unit } ================================================ FILE: ann/src/main/scala/com/twitter/ann/hnsw/JMapBasedIdEmbeddingMap.scala ================================================ package com.twitter.ann.hnsw import com.twitter.ann.common.EmbeddingType.EmbeddingVector import com.twitter.bijection.Injection import com.twitter.search.common.file.AbstractFile import java.io.OutputStream import java.util.concurrent.ConcurrentHashMap import scala.collection.JavaConverters._ private[hnsw] object JMapBasedIdEmbeddingMap { /** * Creates in-memory concurrent hashmap based container that for storing id embedding mapping. * @param expectedElements: Expected num of elements for sizing hint, need not be exact. */ def applyInMemory[T](expectedElements: Int): IdEmbeddingMap[T] = new JMapBasedIdEmbeddingMap[T]( new ConcurrentHashMap[T, EmbeddingVector](expectedElements), Option.empty ) /** * Creates in-memory concurrent hashmap based container that can be serialized to disk for storing id embedding mapping. * @param expectedElements: Expected num of elements for sizing hint, need not be exact. * @param injection : Injection for typed Id T to Array[Byte] */ def applyInMemoryWithSerialization[T]( expectedElements: Int, injection: Injection[T, Array[Byte]] ): IdEmbeddingMap[T] = new JMapBasedIdEmbeddingMap[T]( new ConcurrentHashMap[T, EmbeddingVector](expectedElements), Some(injection) ) /** * Loads id embedding mapping in in-memory concurrent hashmap. * @param embeddingFile: Local/Hdfs file path for embeddings * @param injection : Injection for typed Id T to Array[Byte] * @param numElements: Expected num of elements for sizing hint, need not be exact */ def loadInMemory[T]( embeddingFile: AbstractFile, injection: Injection[T, Array[Byte]], numElements: Option[Int] = Option.empty ): IdEmbeddingMap[T] = { val map = numElements match { case Some(elements) => new ConcurrentHashMap[T, EmbeddingVector](elements) case None => new ConcurrentHashMap[T, EmbeddingVector]() } HnswIOUtil.loadEmbeddings( embeddingFile, injection, new JMapBasedIdEmbeddingMap(map, Some(injection)) ) } } private[this] class JMapBasedIdEmbeddingMap[T]( map: java.util.concurrent.ConcurrentHashMap[T, EmbeddingVector], injection: Option[Injection[T, Array[Byte]]]) extends IdEmbeddingMap[T] { override def putIfAbsent(id: T, embedding: EmbeddingVector): EmbeddingVector = { map.putIfAbsent(id, embedding) } override def put(id: T, embedding: EmbeddingVector): EmbeddingVector = { map.put(id, embedding) } override def get(id: T): EmbeddingVector = { map.get(id) } override def iter(): Iterator[(T, EmbeddingVector)] = map .entrySet() .iterator() .asScala .map(e => (e.getKey, e.getValue)) override def size(): Int = map.size() override def toDirectory(embeddingFile: OutputStream): Unit = { HnswIOUtil.saveEmbeddings(embeddingFile, injection.get, iter()) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/hnsw/MapDbBasedIdEmbeddingMap.scala ================================================ package com.twitter.ann.hnsw import com.twitter.ann.common.EmbeddingType.EmbeddingVector import com.twitter.bijection.Injection import com.twitter.ml.api.embedding.Embedding import com.twitter.search.common.file.AbstractFile import java.io.OutputStream import org.mapdb.DBMaker import org.mapdb.HTreeMap import org.mapdb.Serializer import scala.collection.JavaConverters._ /** * This class currently only support querying and creates map db on fly from thrift serialized embedding mapping * Implement index creation with this or altogether replace mapdb with some better performing solution as it takes a lot of time to create/query or precreate while serializing thrift embeddings */ private[hnsw] object MapDbBasedIdEmbeddingMap { /** * Loads id embedding mapping in mapDB based container leveraging memory mapped files. * @param embeddingFile: Local/Hdfs file path for embeddings * @param injection : Injection for typed Id T to Array[Byte] */ def loadAsReadonly[T]( embeddingFile: AbstractFile, injection: Injection[T, Array[Byte]] ): IdEmbeddingMap[T] = { val diskDb = DBMaker .tempFileDB() .concurrencyScale(32) .fileMmapEnable() .fileMmapEnableIfSupported() .fileMmapPreclearDisable() .cleanerHackEnable() .closeOnJvmShutdown() .make() val mapDb = diskDb .hashMap("mapdb", Serializer.BYTE_ARRAY, Serializer.FLOAT_ARRAY) .createOrOpen() HnswIOUtil.loadEmbeddings( embeddingFile, injection, new MapDbBasedIdEmbeddingMap(mapDb, injection) ) } } private[this] class MapDbBasedIdEmbeddingMap[T]( mapDb: HTreeMap[Array[Byte], Array[Float]], injection: Injection[T, Array[Byte]]) extends IdEmbeddingMap[T] { override def putIfAbsent(id: T, embedding: EmbeddingVector): EmbeddingVector = { val value = mapDb.putIfAbsent(injection.apply(id), embedding.toArray) if (value == null) null else Embedding(value) } override def put(id: T, embedding: EmbeddingVector): EmbeddingVector = { val value = mapDb.put(injection.apply(id), embedding.toArray) if (value == null) null else Embedding(value) } override def get(id: T): EmbeddingVector = { Embedding(mapDb.get(injection.apply(id))) } override def iter(): Iterator[(T, EmbeddingVector)] = { mapDb .entrySet() .iterator() .asScala .map(entry => (injection.invert(entry.getKey).get, Embedding(entry.getValue))) } override def size(): Int = mapDb.size() override def toDirectory(embeddingFile: OutputStream): Unit = { HnswIOUtil.saveEmbeddings(embeddingFile, injection, iter()) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/hnsw/SerializableHnsw.scala ================================================ package com.twitter.ann.hnsw import com.google.common.annotations.VisibleForTesting import com.twitter.ann.common.EmbeddingType.EmbeddingVector import com.twitter.ann.common._ import com.twitter.ann.common.thriftscala.HnswIndexMetadata import com.twitter.ann.hnsw.HnswCommon._ import com.twitter.ann.hnsw.HnswIndex.RandomProvider import com.twitter.bijection.Injection import com.twitter.search.common.file.AbstractFile import com.twitter.search.common.file.FileUtils import com.twitter.util.Future import java.io.IOException import java.util.concurrent.ThreadLocalRandom import java.util.Random import org.apache.beam.sdk.io.fs.ResourceId private[hnsw] object SerializableHnsw { private[hnsw] def apply[T, D <: Distance[D]]( index: Hnsw[T, D], injection: Injection[T, Array[Byte]] ): SerializableHnsw[T, D] = { new SerializableHnsw[T, D]( index, injection ) } private[hnsw] def loadMapBasedQueryableIndex[T, D <: Distance[D]]( dimension: Int, metric: Metric[D], injection: Injection[T, Array[Byte]], futurePool: ReadWriteFuturePool, directory: AbstractFile ): SerializableHnsw[T, D] = { val metadata = HnswIOUtil.loadIndexMetadata(directory.getChild(MetaDataFileName)) validateMetadata(dimension, metric, metadata) val idEmbeddingMap = JMapBasedIdEmbeddingMap.loadInMemory( directory.getChild(EmbeddingMappingFileName), injection, Some(metadata.numElements) ) loadIndex( dimension, metric, injection, futurePool, directory, idEmbeddingMap, metadata ) } private[hnsw] def loadMMappedBasedQueryableIndex[T, D <: Distance[D]]( dimension: Int, metric: Metric[D], injection: Injection[T, Array[Byte]], futurePool: ReadWriteFuturePool, directory: AbstractFile ): SerializableHnsw[T, D] = { val metadata = HnswIOUtil.loadIndexMetadata(directory.getChild(MetaDataFileName)) validateMetadata(dimension, metric, metadata) loadIndex( dimension, metric, injection, futurePool, directory, MapDbBasedIdEmbeddingMap .loadAsReadonly(directory.getChild(EmbeddingMappingFileName), injection), metadata ) } private[hnsw] def loadIndex[T, D <: Distance[D]]( dimension: Int, metric: Metric[D], injection: Injection[T, Array[Byte]], futurePool: ReadWriteFuturePool, directory: AbstractFile, idEmbeddingMap: IdEmbeddingMap[T], metadata: HnswIndexMetadata ): SerializableHnsw[T, D] = { val distFn = DistanceFunctionGenerator(metric, (key: T) => idEmbeddingMap.get(key)) val randomProvider = new RandomProvider { override def get(): Random = ThreadLocalRandom.current() } val internalIndex = HnswIndex.loadHnswIndex[T, EmbeddingVector]( distFn.index, distFn.query, directory.getChild(InternalIndexDir), injection, randomProvider ) val index = new Hnsw[T, D]( dimension, metric, internalIndex, futurePool, idEmbeddingMap, distFn.shouldNormalize, LockedAccess.apply(metadata.numElements) ) new SerializableHnsw(index, injection) } private[this] def validateMetadata[D <: Distance[D]]( dimension: Int, metric: Metric[D], existingMetadata: HnswIndexMetadata ): Unit = { assert( existingMetadata.dimension == dimension, s"Dimensions do not match. requested: $dimension existing: ${existingMetadata.dimension}" ) val existingMetric = Metric.fromThrift(existingMetadata.distanceMetric) assert( existingMetric == metric, s"DistanceMetric do not match. requested: $metric existing: $existingMetric" ) } } @VisibleForTesting private[hnsw] class SerializableHnsw[T, D <: Distance[D]]( index: Hnsw[T, D], injection: Injection[T, Array[Byte]]) extends Appendable[T, HnswParams, D] with Queryable[T, HnswParams, D] with Serialization with Updatable[T] { override def append(entity: EntityEmbedding[T]) = index.append(entity) override def toQueryable: Queryable[T, HnswParams, D] = index.toQueryable override def query( embedding: EmbeddingVector, numOfNeighbours: Int, runtimeParams: HnswParams ) = index.query(embedding, numOfNeighbours, runtimeParams) override def queryWithDistance( embedding: EmbeddingVector, numOfNeighbours: Int, runtimeParams: HnswParams ) = index.queryWithDistance(embedding, numOfNeighbours, runtimeParams) def toDirectory(directory: ResourceId): Unit = { toDirectory(new IndexOutputFile(directory)) } def toDirectory(directory: AbstractFile): Unit = { // Create a temp dir with time prefix, and then do a rename after serialization val tmpDir = FileUtils.getTmpFileHandle(directory) if (!tmpDir.exists()) { tmpDir.mkdirs() } toDirectory(new IndexOutputFile(tmpDir)) // Rename tmp dir to original directory supplied if (!tmpDir.rename(directory)) { throw new IOException(s"Failed to rename ${tmpDir.getPath} to ${directory.getPath}") } } private def toDirectory(indexFile: IndexOutputFile): Unit = { // Save java based hnsw index index.getIndex.toDirectory(indexFile.createDirectory(InternalIndexDir), injection) // Save index metadata HnswIOUtil.saveIndexMetadata( index.getDimen, index.getMetric, index.getIdEmbeddingMap.size(), indexFile.createFile(MetaDataFileName).getOutputStream() ) // Save embedding mapping index.getIdEmbeddingMap.toDirectory( indexFile.createFile(EmbeddingMappingFileName).getOutputStream()) // Create _SUCCESS file indexFile.createSuccessFile() } override def update( entity: EntityEmbedding[T] ): Future[Unit] = { index.update(entity) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/hnsw/TypedHnswIndex.scala ================================================ package com.twitter.ann.hnsw import com.twitter.ann.common._ import com.twitter.bijection.Injection import com.twitter.search.common.file.AbstractFile // Class to provide HNSW based approximate nearest neighbour index object TypedHnswIndex { /** * Creates in-memory HNSW based index which supports querying/addition/updates of the entity embeddings. * See https://docbird.twitter.biz/ann/hnsw.html to check information about arguments. * * @param dimension Dimension of the embedding to be indexed * @param metric Distance metric (InnerProduct/Cosine/L2) * @param efConstruction The parameter has the same meaning as ef, but controls the * index_time/index_accuracy ratio. Bigger ef_construction leads to longer * construction, but better index quality. At some point, increasing * ef_construction does not improve the quality of the index. One way to * check if the selection of ef_construction was ok is to measure a recall * for M nearest neighbor search when ef = ef_constuction: if the recall is * lower than 0.9, than there is room for improvement. * @param maxM The number of bi-directional links created for every new element during construction. * Reasonable range for M is 2-100. Higher M work better on datasets with high * intrinsic dimensionality and/or high recall, while low M work better for datasets * with low intrinsic dimensionality and/or low recalls. The parameter also determines * the algorithm's memory consumption, bigger the param more the memory requirement. * For high dimensional datasets (word embeddings, good face descriptors), higher M * are required (e.g. M=48, 64) for optimal performance at high recall. * The range M=12-48 is ok for the most of the use cases. * @param expectedElements Approximate number of elements to be indexed * @param readWriteFuturePool Future pool for performing read (query) and write operation (addition/updates). * @tparam T Type of item to index * @tparam D Type of distance */ def index[T, D <: Distance[D]]( dimension: Int, metric: Metric[D], efConstruction: Int, maxM: Int, expectedElements: Int, readWriteFuturePool: ReadWriteFuturePool ): Appendable[T, HnswParams, D] with Queryable[T, HnswParams, D] with Updatable[T] = { Hnsw[T, D]( dimension, metric, efConstruction, maxM, expectedElements, readWriteFuturePool, JMapBasedIdEmbeddingMap.applyInMemory[T](expectedElements) ) } /** * Creates in-memory HNSW based index which supports querying/addition/updates of the entity embeddings. * It can be serialized to a directory (HDFS/Local file system) * See https://docbird.twitter.biz/ann/hnsw.html to check information about arguments. * * @param dimension Dimension of the embedding to be indexed * @param metric Distance metric (InnerProduct/Cosine/L2) * @param efConstruction The parameter has the same meaning as ef, but controls the * index_time/index_accuracy ratio. Bigger ef_construction leads to longer * construction, but better index quality. At some point, increasing * ef_construction does not improve the quality of the index. One way to * check if the selection of ef_construction was ok is to measure a recall * for M nearest neighbor search when ef = ef_constuction: if the recall is * lower than 0.9, than there is room for improvement. * @param maxM The number of bi-directional links created for every new element during construction. * Reasonable range for M is 2-100. Higher M work better on datasets with high * intrinsic dimensionality and/or high recall, while low M work better for datasets * with low intrinsic dimensionality and/or low recalls. The parameter also determines * the algorithm's memory consumption, bigger the param more the memory requirement. * For high dimensional datasets (word embeddings, good face descriptors), higher M * are required (e.g. M=48, 64) for optimal performance at high recall. * The range M=12-48 is ok for the most of the use cases. * @param expectedElements Approximate number of elements to be indexed * @param injection Injection for typed Id T to Array[Byte] * @param readWriteFuturePool Future pool for performing read (query) and write operation (addition/updates). * @tparam T Type of item to index * @tparam D Type of distance */ def serializableIndex[T, D <: Distance[D]]( dimension: Int, metric: Metric[D], efConstruction: Int, maxM: Int, expectedElements: Int, injection: Injection[T, Array[Byte]], readWriteFuturePool: ReadWriteFuturePool ): Appendable[T, HnswParams, D] with Queryable[T, HnswParams, D] with Updatable[T] with Serialization = { val index = Hnsw[T, D]( dimension, metric, efConstruction, maxM, expectedElements, readWriteFuturePool, JMapBasedIdEmbeddingMap .applyInMemoryWithSerialization[T](expectedElements, injection) ) SerializableHnsw[T, D]( index, injection ) } /** * Loads HNSW index from a directory to in-memory * @param dimension dimension of the embedding to be indexed * @param metric Distance metric * @param readWriteFuturePool Future pool for performing read (query) and write operation (addition/updates). * @param injection : Injection for typed Id T to Array[Byte] * @param directory : Directory(HDFS/Local file system) where hnsw index is stored * @tparam T : Type of item to index * @tparam D : Type of distance */ def loadIndex[T, D <: Distance[D]]( dimension: Int, metric: Metric[D], injection: Injection[T, Array[Byte]], readWriteFuturePool: ReadWriteFuturePool, directory: AbstractFile ): Appendable[T, HnswParams, D] with Queryable[T, HnswParams, D] with Updatable[T] with Serialization = { SerializableHnsw.loadMapBasedQueryableIndex[T, D]( dimension, metric, injection, readWriteFuturePool, directory ) } /** * Loads a HNSW index from a directory and memory map it. * It will take less memory but rely more on disk as it leverages memory mapped file backed by disk. * Latency will go up considerably (Could be by factor of > 10x) if used on instance with low * memory since lot of page faults may occur. Best use case to use would with scalding jobs * where mapper/reducers instance are limited by 8gb memory. * @param dimension dimension of the embedding to be indexed * @param metric Distance metric * @param readWriteFuturePool Future pool for performing read (query) and write operation (addition/updates). * @param injection Injection for typed Id T to Array[Byte] * @param directory Directory(HDFS/Local file system) where hnsw index is stored * @tparam T Type of item to index * @tparam D Type of distance */ def loadMMappedIndex[T, D <: Distance[D]]( dimension: Int, metric: Metric[D], injection: Injection[T, Array[Byte]], readWriteFuturePool: ReadWriteFuturePool, directory: AbstractFile ): Appendable[T, HnswParams, D] with Queryable[T, HnswParams, D] with Updatable[T] with Serialization = { SerializableHnsw.loadMMappedBasedQueryableIndex[T, D]( dimension, metric, injection, readWriteFuturePool, directory ) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/manhattan/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/bijection:core", "3rdparty/jvm/com/twitter/bijection:scrooge", "ann/src/main/scala/com/twitter/ann/common", "src/scala/com/twitter/ml/api/embedding", "storage/clients/manhattan", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/manhattan/ManhattanEmbeddingProducer.scala ================================================ package com.twitter.ann.manhattan import com.twitter.ann.common.EmbeddingType.EmbeddingVector import com.twitter.ann.common.{EmbeddingProducer, EmbeddingType} import com.twitter.bijection.Injection import com.twitter.ml.api.embedding.{EmbeddingBijection, EmbeddingSerDe} import com.twitter.ml.api.{thriftscala => thrift} import com.twitter.stitch.Stitch import com.twitter.storage.client.manhattan.bijections.Bijections import com.twitter.storage.client.manhattan.bijections.Bijections.BinaryScalaInjection import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint import com.twitter.storage.client.manhattan.kv.impl.{ DescriptorP1L0, ReadOnlyKeyDescriptor, ValueDescriptor } private[manhattan] class ManhattanEmbeddingProducer[T]( keyDescriptor: DescriptorP1L0.DKey[T], valueDescriptor: ValueDescriptor.EmptyValue[EmbeddingVector], manhattanEndpoint: ManhattanKVEndpoint) extends EmbeddingProducer[T] { /** * Lookup an embedding from manhattan given a key of type T. * * @return An embedding stitch. * An easy way to get a Future from a Stitch is to run Stitch.run(stitch) */ override def produceEmbedding(input: T): Stitch[Option[EmbeddingVector]] = { val fullKey = keyDescriptor.withPkey(input) val stitchResult = manhattanEndpoint.get(fullKey, valueDescriptor) stitchResult.map { resultOption => resultOption.map(_.contents) } } } object ManhattanEmbeddingProducer { private[manhattan] def keyDescriptor[T]( injection: Injection[T, Array[Byte]], dataset: String ): DescriptorP1L0.DKey[T] = ReadOnlyKeyDescriptor(injection.andThen(Bijections.BytesBijection)) .withDataset(dataset) private[manhattan] val EmbeddingDescriptor: ValueDescriptor.EmptyValue[ EmbeddingType.EmbeddingVector ] = { val embeddingBijection = new EmbeddingBijection(EmbeddingSerDe.floatEmbeddingSerDe) val thriftInjection = BinaryScalaInjection[thrift.Embedding](thrift.Embedding) ValueDescriptor(embeddingBijection.andThen(thriftInjection)) } def apply[T]( dataset: String, injection: Injection[T, Array[Byte]], manhattanEndpoint: ManhattanKVEndpoint ): EmbeddingProducer[T] = { val descriptor = keyDescriptor(injection, dataset) new ManhattanEmbeddingProducer(descriptor, EmbeddingDescriptor, manhattanEndpoint) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/manhattan/README ================================================ # Description The ManhattanEmbeddingProducer is an EmbeddingProducer that is backed by a static manhattan dataset. # Setting up Data Data needs to be setup correctly in manhattan in order to be able to read the data using the ManhattanEmbeddingProducer. You can use the EmbeddingSamplingJob to do this. The job can reads embedding data from HDFS and re-writes it in the manhattan data format on HDFS. ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/benchmark/BUILD ================================================ scala_library( name = "benchmark", sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = [ "bazel-compatible", "bazel-compatible:migrated", "bazel-only", ], dependencies = [ ":user_item_knn-scala", "3rdparty/src/jvm/com/twitter/scalding:args", "3rdparty/src/jvm/com/twitter/scalding:core", "3rdparty/src/jvm/com/twitter/scalding:date", "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/scala/com/twitter/ann/scalding/offline", "src/scala/com/twitter/scalding_internal/dalv2", "src/scala/com/twitter/scalding_internal/job", "src/scala/com/twitter/scalding_internal/job/analytics_batch", "src/scala/com/twitter/scalding_internal/multiformat/format", ], ) hadoop_binary( name = "benchmark-adhoc", main = "com.twitter.scalding.Tool", platform = "java8", runtime_platform = "java8", tags = [ "bazel-compatible", "bazel-compatible:migrated", "bazel-only", ], dependencies = [ ":benchmark", "3rdparty/jvm/org/slf4j:slf4j-jdk14", ], ) create_datasets( base_name = "user_item_knn", description = "List of the top recommendations per search entity (user)", java_schema = "com.twitter.ann.knn.thriftjava.Knn", platform = "java8", role = "cortex-mlx", scala_schema = "com.twitter.ann.knn.thriftscala.Knn", segment_type = "partitioned", tags = ["bazel-compatible"], java_dependencies = [ "ann/src/main/thrift/com/twitter/ann/knn:thrift-java", ], scala_dependencies = [ "ann/src/main/thrift/com/twitter/ann/knn:thrift-scala", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/benchmark/Knn.scala ================================================ package com.twitter.ann.scalding.offline.com.twitter.ann.scalding.benchmark /* This job will generate KNN ground truth based user and item embeddings. */ import com.twitter.scalding.typed.TypedPipe import com.twitter.scalding._ import com.twitter.scalding_internal.dalv2.DALWrite.D import com.twitter.ann.knn.thriftscala.Knn import com.twitter.ann.knn.thriftscala.Neighbor import com.twitter.ann.scalding.offline.IndexingStrategy import com.twitter.ann.scalding.offline.KnnHelper import com.twitter.ann.common.Distance import com.twitter.ml.featurestore.lib.embedding.EmbeddingWithEntity import com.twitter.cortex.ml.embeddings.common.EmbeddingFormatArgsParser import com.twitter.cortex.ml.embeddings.common.EntityKind import java.util.TimeZone import com.twitter.scalding_internal.dalv2.DALWrite._ import com.twitter.ann.scalding.benchmark.UserItemKnnScalaDataset import com.twitter.scalding_internal.job.TwitterExecutionApp import com.twitter.ml.featurestore.lib.EntityId import com.twitter.ml.featurestore.lib.UserId /** * This job will take consumer and item embeddings(either url or tweet) and output Knn entities (user id, (distance, item id)). * * Example command to run this adhoc job: * * scalding remote run \ * --target ann/src/main/scala/com/twitter/ann/scalding/benchmark:benchmark-adhoc \ * --hadoop-properties "mapreduce.map.memory.mb=8192 mapreduce.map.java.opts='-Xmx7618M' mapreduce.reduce.memory.mb=8192 mapreduce.reduce.java.opts='-Xmx7618M' mapred.task.timeout=0" \ * --submitter hadoopnest3.smf1.twitter.com \ * --user cortex-mlx \ * --submitter-memory 8000.megabyte \ * --main-class com.twitter.ann.scalding.offline.com.twitter.ann.scalding.benchmark.KnnJob -- \ * --dalEnvironment Prod \ * --search_space_entity_type user \ * --user.feature_store_embedding ConsumerFollowEmbedding300Dataset \ * --user.feature_store_major_version 1569196895 \ * --user.date_range 2019-10-23 \ * --search_space.feature_store_embedding ConsumerFollowEmbedding300Dataset \ * --search_space.feature_store_major_version 1569196895 \ * --search_space.date_range 2019-10-23 \ * --date 2019-10-25 \ * --version "consumer_follower_test" \ * --reducers 10000 \ * --num_of_random_groups 20 \ * --num_replicas 1000 \ * --indexing_strategy.metric InnerProduct \ * --indexing_strategy.type hnsw \ * --indexing_strategy.dimension 300 \ * --indexing_strategy.ef_construction 30 \ * --indexing_strategy.max_m 10 \ * --indexing_strategy.ef_query 50 \ * --search_space_shards 3000 \ * --query_shards 3000 \ * --search_space.read_sample_ratio 0.038 */ trait KnnJobBase { val seed: Long = 123 def getKnnDataset[B <: EntityId, D <: Distance[D]]( args: Args )( implicit uniqueID: UniqueID ): TypedPipe[Knn] = { val consumerPipe: TypedPipe[EmbeddingWithEntity[UserId]] = EmbeddingFormatArgsParser.User .getEmbeddingFormat(args, "user") .getEmbeddings val itemPipe = EntityKind .getEntityKind(args("search_space_entity_type")) .parser .getEmbeddingFormat(args, "search_space") .getEmbeddings KnnHelper // Refer to the documentation of findNearestNeighboursWithIndexingStrategy for more // information about how to set these settings. .findNearestNeighboursWithIndexingStrategy[UserId, B, D]( queryEmbeddings = consumerPipe, searchSpaceEmbeddings = itemPipe.asInstanceOf[TypedPipe[EmbeddingWithEntity[B]]], numNeighbors = args.int("candidate_per_user", 20), reducersOption = args.optional("reducers").map(_.toInt), numOfSearchGroups = args.int("num_of_random_groups"), numReplicas = args.int("num_replicas"), indexingStrategy = IndexingStrategy.parse(args).asInstanceOf[IndexingStrategy[D]], queryShards = args.optional("query_shards").map(_.toInt), searchSpaceShards = args.optional("search_space_shards").map(_.toInt) ) .map { case (user, items) => val neighbors = items.map { case (item, distance) => Neighbor( distance.distance, item.toThrift ) } Knn(user.toThrift, neighbors) } } } object KnnJob extends TwitterExecutionApp with KnnJobBase { val KnnPathSuffix: String = "/user/cortex-mlx/qualatative_analysis/knn_ground_truth/" val partitionKey: String = "version" override def job: Execution[Unit] = Execution.withId { implicit uniqueId => Execution.getArgs.flatMap { args: Args => implicit val timeZone: TimeZone = TimeZone.getDefault implicit val dateParser: DateParser = DateParser.default implicit val dateRange: DateRange = DateRange.parse(args.list("date"))(timeZone, dateParser) getKnnDataset(args).writeDALExecution( UserItemKnnScalaDataset, D.Daily, D.Suffix(KnnPathSuffix), D.Parquet, Set(D.Partition(partitionKey, args("version"), D.PartitionType.String)) ) } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = [ "bazel-compatible", "bazel-only", ], dependencies = [ "3rdparty/jvm/com/twitter/bijection:scrooge", "3rdparty/src/jvm/com/twitter/scalding:args", "3rdparty/src/jvm/com/twitter/scalding:commons", "3rdparty/src/jvm/com/twitter/scalding:core", "ann/src/main/scala/com/twitter/ann/brute_force", "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/scala/com/twitter/ann/hnsw", "ann/src/main/scala/com/twitter/ann/util", "cortex-core/entity-embeddings/src/thrift/com/twitter/entityembeddings/neighbors:embeddings-knn-thrift-scala", "src/scala/com/twitter/cortex/ml/embeddings/common:Helpers-deploy", "src/scala/com/twitter/pluck/source/core_workflows/user_model:condensed_user_state-scala", "src/scala/com/twitter/scalding_internal/dalv2", "src/scala/com/twitter/scalding_internal/job", "src/scala/com/twitter/scalding_internal/multiformat/format", "src/scala/com/twitter/scalding_internal/parquet_thrift", "usersource/snapshot/src/main/scala/com/twitter/usersource/snapshot/flat:usersource_flat-scala", "usersource/snapshot/src/main/thrift/com/twitter/usersource/snapshot/flat:flat-scala", ], ) hadoop_binary( name = "ann-offline-deploy", main = "com.twitter.scalding.Tool", platform = "java8", runtime_platform = "java8", tags = [ "bazel-compatible", "bazel-compatible:migrated", "bazel-only", ], dependencies = [ ":offline", "3rdparty/jvm/org/slf4j:slf4j-jdk14", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/IndexingStrategy.scala ================================================ package com.twitter.ann.scalding.offline import com.twitter.ann.brute_force.{BruteForceIndex, BruteForceRuntimeParams} import com.twitter.ann.common.{Distance, EntityEmbedding, Metric, ReadWriteFuturePool} import com.twitter.ann.hnsw.{HnswParams, TypedHnswIndex} import com.twitter.ann.util.IndexBuilderUtils import com.twitter.scalding.Args import com.twitter.util.logging.Logger import com.twitter.util.{Await, FuturePool} /** * IndexingStrategy is used for determining how we will build the index when doing a KNN in * scalding. Right now there are 2 strategies a BruteForce and HNSW strategy. * @tparam D distance that the index uses. */ sealed trait IndexingStrategy[D <: Distance[D]] { private[offline] def buildIndex[T]( indexItems: TraversableOnce[EntityEmbedding[T]] ): ParameterlessQueryable[T, _, D] } object IndexingStrategy { /** * Parse an indexing strategy from scalding args. * ${argumentName}.type Is hsnw or brute_force * ${argumentName}.type is the metric to use. See Metric.fromString for options. * * hsnw has these additional parameters: * ${argumentName}.dimension the number of dimension for the embeddings. * ${argumentName}.ef_construction, ${argumentName}.ef_construction and ${argumentName}.ef_query. * See TypedHnswIndex for more details on these parameters. * @param args scalding arguments to parse. * @param argumentName A specifier to use in case you want to parse more than one indexing * strategy. indexing_strategy by default. * @return parse indexing strategy */ def parse( args: Args, argumentName: String = "indexing_strategy" ): IndexingStrategy[_] = { def metricArg[D <: Distance[D]] = Metric.fromString(args(s"$argumentName.metric")).asInstanceOf[Metric[D]] args(s"$argumentName.type") match { case "brute_force" => BruteForceIndexingStrategy(metricArg) case "hnsw" => val dimensionArg = args.int(s"$argumentName.dimension") val efConstructionArg = args.int(s"$argumentName.ef_construction") val maxMArg = args.int(s"$argumentName.max_m") val efQuery = args.int(s"$argumentName.ef_query") HnswIndexingStrategy( dimension = dimensionArg, metric = metricArg, efConstruction = efConstructionArg, maxM = maxMArg, hnswParams = HnswParams(efQuery) ) } } } case class BruteForceIndexingStrategy[D <: Distance[D]](metric: Metric[D]) extends IndexingStrategy[D] { private[offline] def buildIndex[T]( indexItems: TraversableOnce[EntityEmbedding[T]] ): ParameterlessQueryable[T, _, D] = { val appendable = BruteForceIndex[T, D](metric, FuturePool.immediatePool) indexItems.foreach { item => Await.result(appendable.append(item)) } val queryable = appendable.toQueryable ParameterlessQueryable[T, BruteForceRuntimeParams.type, D]( queryable, BruteForceRuntimeParams ) } } case class HnswIndexingStrategy[D <: Distance[D]]( dimension: Int, metric: Metric[D], efConstruction: Int, maxM: Int, hnswParams: HnswParams, concurrencyLevel: Int = 1) extends IndexingStrategy[D] { private[offline] def buildIndex[T]( indexItems: TraversableOnce[EntityEmbedding[T]] ): ParameterlessQueryable[T, _, D] = { val log: Logger = Logger(getClass) val appendable = TypedHnswIndex.index[T, D]( dimension = dimension, metric = metric, efConstruction = efConstruction, maxM = maxM, // This is not really that important. expectedElements = 1000, readWriteFuturePool = ReadWriteFuturePool(FuturePool.immediatePool) ) val future = IndexBuilderUtils .addToIndex(appendable, indexItems.toStream, concurrencyLevel) .map { numberUpdates => log.info(s"Performed $numberUpdates updates") } Await.result(future) val queryable = appendable.toQueryable ParameterlessQueryable( queryable, hnswParams ) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/KnnDebug.scala ================================================ package com.twitter.ann.scalding.offline import com.twitter.core_workflows.user_model.thriftscala.CondensedUserState import com.twitter.cortex.ml.embeddings.common.{DataSourceManager, GraphEdge, Helpers, UserKind} import com.twitter.ml.featurestore.lib.UserId import com.twitter.entityembeddings.neighbors.thriftscala.{EntityKey, NearestNeighbors} import com.twitter.pluck.source.core_workflows.user_model.CondensedUserStateScalaDataset import com.twitter.scalding._ import com.twitter.scalding.typed.TypedPipe import com.twitter.scalding_internal.dalv2.DAL import com.twitter.usersource.snapshot.flat.UsersourceFlatScalaDataset import com.twitter.usersource.snapshot.flat.thriftscala.FlatUser case class ConsumerAssoc(consumerId: UserId, assoc: List[String]) object KnnDebug { def getConsumerAssociations( graph: TypedPipe[GraphEdge[UserId, UserId]], usernames: TypedPipe[(UserId, String)], reducers: Int ): TypedPipe[ConsumerAssoc] = { graph .groupBy(_.itemId) .join(usernames).withReducers(reducers) .values .map { case (edge: GraphEdge[UserId, UserId], producerScreenName: String) => ConsumerAssoc(consumerId = edge.consumerId, assoc = List(producerScreenName)) } .groupBy(_.consumerId).withReducers(reducers) .reduce[ConsumerAssoc] { case (uFollow1: ConsumerAssoc, uFollow2: ConsumerAssoc) => ConsumerAssoc(consumerId = uFollow1.consumerId, assoc = uFollow1.assoc ++ uFollow2.assoc) } .values } /** * Write the neighbors and a set of follows to a tsv for easier analysis during debugging * We take the set of users with between 25-50 follows and grab only those users * * This returns 4 strings of the form: * consumerId, state, followUserNamefollowUserNamefollowUserName, neighborNameneighborNameneighborName */ def getDebugTable( neighborsPipe: TypedPipe[(EntityKey, NearestNeighbors)], shards: Int, reducers: Int, limit: Int = 10000, userDataset: Option[TypedPipe[FlatUser]] = None, followDataset: Option[TypedPipe[GraphEdge[UserId, UserId]]] = None, consumerStatesDataset: Option[TypedPipe[CondensedUserState]] = None, minFollows: Int = 25, maxFollows: Int = 50 )( implicit dateRange: DateRange ): TypedPipe[(String, String, String, String)] = { val usersourcePipe: TypedPipe[FlatUser] = userDataset .getOrElse(DAL.readMostRecentSnapshot(UsersourceFlatScalaDataset, dateRange).toTypedPipe) val followGraph: TypedPipe[GraphEdge[UserId, UserId]] = followDataset .getOrElse(new DataSourceManager().getFollowGraph()) val consumerStates: TypedPipe[CondensedUserState] = consumerStatesDataset .getOrElse(DAL.read(CondensedUserStateScalaDataset).toTypedPipe) val usernames: TypedPipe[(UserId, String)] = usersourcePipe.flatMap { flatUser => (flatUser.screenName, flatUser.id) match { case (Some(name: String), Some(userId: Long)) => Some((UserId(userId), name)) case _ => None } }.fork val consumerFollows: TypedPipe[ConsumerAssoc] = getConsumerAssociations(followGraph, usernames, reducers) .filter { uFollow => (uFollow.assoc.size > minFollows && uFollow.assoc.size < maxFollows) } val neighborGraph: TypedPipe[GraphEdge[UserId, UserId]] = neighborsPipe .limit(limit) .flatMap { case (entityKey: EntityKey, neighbors: NearestNeighbors) => Helpers.optionalToLong(entityKey.id) match { case Some(entityId: Long) => neighbors.neighbors.flatMap { neighbor => Helpers .optionalToLong(neighbor.neighbor.id) .map { neighborId => GraphEdge[UserId, UserId]( consumerId = UserId(entityId), itemId = UserId(neighborId), weight = 1.0F) } } case None => List() } } val consumerNeighbors: TypedPipe[ConsumerAssoc] = getConsumerAssociations(neighborGraph, usernames, reducers) consumerFollows .groupBy(_.consumerId) .join(consumerStates.groupBy { consumer => UserId(consumer.uid) }).withReducers(reducers) .join(consumerNeighbors.groupBy(_.consumerId)).withReducers(reducers) .values .map { case ((uFollow: ConsumerAssoc, state: CondensedUserState), uNeighbors: ConsumerAssoc) => ( UserKind.stringInjection(uFollow.consumerId), state.state.toString, uFollow.assoc mkString "", uNeighbors.assoc mkString "") } .shard(shards) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/KnnEntityRecoDebugJob.scala ================================================ package com.twitter.ann.scalding.offline import com.twitter.ann.common.Distance import com.twitter.ann.common.Metric import com.twitter.cortex.ml.embeddings.common.EntityKind import com.twitter.ml.featurestore.lib.EntityId import com.twitter.scalding.typed.TypedPipe import com.twitter.scalding._ import com.twitter.scalding_internal.job.TwitterExecutionApp /** * This job do an exhaustive search for nearest neighbours helpful for debugging recommendations * for a given list of sample queryIds and entity embeddings for the recos to be made. * Sample job script: ./bazel bundle ann/src/main/scala/com/twitter/ann/scalding/offline:ann-offline-deploy oscar hdfs \ --screen --tee log.txt \ --hadoop-client-memory 6000 \ --hadoop-properties "yarn.app.mapreduce.am.resource.mb=6000;yarn.app.mapreduce.am.command-opts='-Xmx7500m';mapreduce.map.memory.mb=7500;mapreduce.reduce.java.opts='-Xmx6000m';mapreduce.reduce.memory.mb=7500;mapred.task.timeout=36000000;" \ --bundle ann-offline-deploy \ --min-split-size 284217728 \ --host hadoopnest1.smf1.twitter.com \ --tool com.twitter.ann.scalding.offline.KnnEntityRecoDebugJob -- \ --neighbors 10 \ --metric InnerProduct \ --query_entity_kind user \ --search_space_entity_kind user \ --query.embedding_path /user/apoorvs/sample_embeddings \ --query.embedding_format tab \ --search_space.embedding_path /user/apoorvs/sample_embeddings \ --search_space.embedding_format tab \ --query_ids 974308319300149248 988871266244464640 2719685122 2489777564 \ --output_path /user/apoorvs/adhochadoop/test \ --reducers 100 */ object KnnEntityRecoDebugJob extends TwitterExecutionApp { override def job: Execution[Unit] = Execution.withId { implicit uniqueId => Execution.getArgs.flatMap { args: Args => val queryEntityKind = EntityKind.getEntityKind(args("query_entity_kind")) val searchSpaceEntityKind = EntityKind.getEntityKind(args("search_space_entity_kind")) val metric = Metric.fromString(args("metric")) run(queryEntityKind, searchSpaceEntityKind, metric, args) } } private[this] def run[A <: EntityId, B <: EntityId, D <: Distance[D]]( uncastQueryEntityKind: EntityKind[_], uncastSearchSpaceEntityKind: EntityKind[_], uncastMetric: Metric[_], args: Args )( implicit uniqueID: UniqueID ): Execution[Unit] = { import KnnHelper._ val numNeighbors = args.int("neighbors") val reducers = args.getOrElse("reducers", "100").toInt val queryEntityKind = uncastQueryEntityKind.asInstanceOf[EntityKind[A]] val searchSpaceEntityKind = uncastSearchSpaceEntityKind.asInstanceOf[EntityKind[B]] val metric = uncastMetric.asInstanceOf[Metric[D]] // Filter the query entity embeddings with the queryIds val queryIds = args.list("query_ids") assert(queryIds.nonEmpty) val filterQueryIds: TypedPipe[A] = TypedPipe .from(queryIds) .map(queryEntityKind.stringInjection.invert(_).get) val queryEmbeddings = queryEntityKind.parser.getEmbeddingFormat(args, "query").getEmbeddings // Get the neighbour embeddings val searchSpaceEmbeddings = searchSpaceEntityKind.parser.getEmbeddingFormat(args, "search_space").getEmbeddings val nearestNeighborString = findNearestNeighbours( queryEmbeddings, searchSpaceEmbeddings, metric, numNeighbors, Some(filterQueryIds), reducers )(queryEntityKind.ordering, uniqueID).map( nearestNeighborsToString(_, queryEntityKind, searchSpaceEntityKind) ) // Write the nearest neighbor string to one part file. nearestNeighborString .shard(1) .writeExecution(TypedTsv(args("output_path"))) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/KnnHelper.scala ================================================ package com.twitter.ann.scalding.offline import com.twitter.ann.common._ import com.twitter.ann.hnsw.{HnswParams, TypedHnswIndex} import com.twitter.bijection.Injection import com.twitter.cortex.ml.embeddings.common.{EntityKind, Helpers, UserKind} import com.twitter.entityembeddings.neighbors.thriftscala.{EntityKey, NearestNeighbors, Neighbor} import com.twitter.ml.api.embedding.Embedding import com.twitter.ml.api.embedding.EmbeddingMath.{Float => math} import com.twitter.ml.featurestore.lib.embedding.EmbeddingWithEntity import com.twitter.ml.featurestore.lib.{EntityId, UserId} import com.twitter.scalding.typed.{TypedPipe, UnsortedGrouped} import com.twitter.scalding.{Args, DateRange, Stat, TextLine, UniqueID} import com.twitter.search.common.file.AbstractFile import com.twitter.util.{Await, FuturePool} import scala.util.Random case class Index[T, D <: Distance[D]]( injection: Injection[T, Array[Byte]], metric: Metric[D], dimension: Int, directory: AbstractFile) { lazy val annIndex = TypedHnswIndex.loadIndex[T, D]( dimension, metric, injection, ReadWriteFuturePool(FuturePool.immediatePool), directory ) } object KnnHelper { def getFilteredUserEmbeddings( args: Args, filterPath: Option[String], reducers: Int, useHashJoin: Boolean )( implicit dateRange: DateRange ): TypedPipe[EmbeddingWithEntity[UserId]] = { val userEmbeddings: TypedPipe[EmbeddingWithEntity[UserId]] = UserKind.parser.getEmbeddingFormat(args, "consumer").getEmbeddings filterPath match { case Some(fileName: String) => val filterUserIds: TypedPipe[UserId] = TypedPipe .from(TextLine(fileName)) .flatMap { idLine => Helpers.optionalToLong(idLine) } .map { id => UserId(id) } Helpers .adjustableJoin( left = userEmbeddings.groupBy(_.entityId), right = filterUserIds.asKeys, useHashJoin = useHashJoin, reducers = Some(reducers) ).map { case (_, (embedding, _)) => embedding } case None => userEmbeddings } } def getNeighborsPipe[T <: EntityId, D <: Distance[D]]( args: Args, uncastEntityKind: EntityKind[_], uncastMetric: Metric[_], ef: Int, consumerEmbeddings: TypedPipe[EmbeddingWithEntity[UserId]], abstractFile: Option[AbstractFile], reducers: Int, numNeighbors: Int, dimension: Int )( implicit dateRange: DateRange ): TypedPipe[(EntityKey, NearestNeighbors)] = { val entityKind = uncastEntityKind.asInstanceOf[EntityKind[T]] val injection = entityKind.byteInjection val metric = uncastMetric.asInstanceOf[Metric[D]] abstractFile match { case Some(directory: AbstractFile) => val index = Index(injection, metric, dimension, directory) consumerEmbeddings .map { embedding => val knn = Await.result( index.annIndex.queryWithDistance( Embedding(embedding.embedding.toArray), numNeighbors, HnswParams(ef) ) ) val neighborList = knn .filter(_.neighbor.toString != embedding.entityId.userId.toString) .map(nn => Neighbor( neighbor = EntityKey(nn.neighbor.toString), similarity = Some(1 - nn.distance.distance))) EntityKey(embedding.entityId.toString) -> NearestNeighbors(neighborList) } case None => val producerEmbeddings: TypedPipe[EmbeddingWithEntity[UserId]] = UserKind.parser.getEmbeddingFormat(args, "producer").getEmbeddings bruteForceNearestNeighbors( consumerEmbeddings, producerEmbeddings, numNeighbors, reducers ) } } def bruteForceNearestNeighbors( consumerEmbeddings: TypedPipe[EmbeddingWithEntity[UserId]], producerEmbeddings: TypedPipe[EmbeddingWithEntity[UserId]], numNeighbors: Int, reducers: Int ): TypedPipe[(EntityKey, NearestNeighbors)] = { consumerEmbeddings .cross(producerEmbeddings) .map { case (cEmbed: EmbeddingWithEntity[UserId], pEmbed: EmbeddingWithEntity[UserId]) => // Cosine similarity val cEmbedNorm = math.l2Norm(cEmbed.embedding).toFloat val pEmbedNorm = math.l2Norm(pEmbed.embedding).toFloat val distance: Float = -math.dotProduct( (math.scalarProduct(cEmbed.embedding, 1 / cEmbedNorm)), math.scalarProduct(pEmbed.embedding, 1 / pEmbedNorm)) ( UserKind.stringInjection(cEmbed.entityId), (distance, UserKind.stringInjection(pEmbed.entityId))) } .groupBy(_._1).withReducers(reducers) .sortWithTake(numNeighbors) { case ((_: String, (sim1: Float, _: String)), (_: String, (sim2: Float, _: String))) => sim1 < sim2 } .map { case (consumerId: String, (prodSims: Seq[(String, (Float, String))])) => EntityKey(consumerId) -> NearestNeighbors( prodSims.map { case (consumerId: String, (sim: Float, prodId: String)) => Neighbor(neighbor = EntityKey(prodId), similarity = Some(-sim.toDouble)) } ) } } /** * Calculate the nearest neighbors exhaustively between two entity embeddings using one as query and other as the search space. * @param queryEmbeddings entity embeddings for queries * @param searchSpaceEmbeddings entity embeddings for search space * @param metric distance metric * @param numNeighbors number of neighbors * @param queryIdsFilter optional query ids to filter to query entity embeddings * @param reducers number of reducers for grouping * @param isSearchSpaceLarger Used for optimization: Is the search space larger than the query space? Ignored if numOfSearchGroups > 1. * @param numOfSearchGroups we divide the search space into these groups (randomly). Useful when the search space is too large. Overrides isSearchSpaceLarger. * @param numReplicas Each search group will be responsible for 1/numReplicas queryEmebeddings. * This might speed up the search when the size of the index embeddings is * large. * @tparam A type of query entity * @tparam B type of search space entity * @tparam D type of distance */ def findNearestNeighbours[A <: EntityId, B <: EntityId, D <: Distance[D]]( queryEmbeddings: TypedPipe[EmbeddingWithEntity[A]], searchSpaceEmbeddings: TypedPipe[EmbeddingWithEntity[B]], metric: Metric[D], numNeighbors: Int = 10, queryIdsFilter: Option[TypedPipe[A]] = Option.empty, reducers: Int = 100, mappers: Int = 100, isSearchSpaceLarger: Boolean = true, numOfSearchGroups: Int = 1, numReplicas: Int = 1, useCounters: Boolean = true )( implicit ordering: Ordering[A], uid: UniqueID ): TypedPipe[(A, Seq[(B, D)])] = { val filteredQueryEmbeddings = queryIdsFilter match { case Some(filter) => { queryEmbeddings.groupBy(_.entityId).hashJoin(filter.asKeys).map { case (x, (embedding, _)) => embedding } } case None => queryEmbeddings } if (numOfSearchGroups > 1) { val indexingStrategy = BruteForceIndexingStrategy(metric) findNearestNeighboursWithIndexingStrategy( queryEmbeddings, searchSpaceEmbeddings, numNeighbors, numOfSearchGroups, indexingStrategy, numReplicas, Some(reducers), useCounters = useCounters ) } else { findNearestNeighboursViaCross( filteredQueryEmbeddings, searchSpaceEmbeddings, metric, numNeighbors, reducers, mappers, isSearchSpaceLarger) } } /** * Calculate the nearest neighbors using the specified indexing strategy between two entity * embeddings using one as query and other as the search space. * @param queryEmbeddings entity embeddings for queries * @param searchSpaceEmbeddings entity embeddings for search space. You should be able to fit * searchSpaceEmbeddings.size / numOfSearchGroups into memory. * @param numNeighbors number of neighbors * @param reducersOption number of reducers for the final sortedTake. * @param numOfSearchGroups we divide the search space into these groups (randomly). Useful when * the search space is too large. Search groups are shards. Choose this * number by ensuring searchSpaceEmbeddings.size / numOfSearchGroups * embeddings will fit into memory. * @param numReplicas Each search group will be responsible for 1/numReplicas queryEmebeddings. * By increasing this number, we can parallelize the work and reduce end to end * running times. * @param indexingStrategy How we will search for nearest neighbors within a search group * @param queryShards one step we have is to fan out the query embeddings. We create one entry * per search group. If numOfSearchGroups is large, then this fan out can take * a long time. You can shard the query shard first to parallelize this * process. One way to estimate what value to use: * queryEmbeddings.size * numOfSearchGroups / queryShards should be around 1GB. * @param searchSpaceShards this param is similar to queryShards. Except it shards the search * space when numReplicas is too large. One way to estimate what value * to use: searchSpaceEmbeddings.size * numReplicas / searchSpaceShards * should be around 1GB. * @tparam A type of query entity * @tparam B type of search space entity * @tparam D type of distance * @return a pipe keyed by the index embedding. The values are the list of numNeighbors nearest * neighbors along with their distances. */ def findNearestNeighboursWithIndexingStrategy[A <: EntityId, B <: EntityId, D <: Distance[D]]( queryEmbeddings: TypedPipe[EmbeddingWithEntity[A]], searchSpaceEmbeddings: TypedPipe[EmbeddingWithEntity[B]], numNeighbors: Int, numOfSearchGroups: Int, indexingStrategy: IndexingStrategy[D], numReplicas: Int = 1, reducersOption: Option[Int] = None, queryShards: Option[Int] = None, searchSpaceShards: Option[Int] = None, useCounters: Boolean = true )( implicit ordering: Ordering[A], uid: UniqueID ): UnsortedGrouped[A, Seq[(B, D)]] = { implicit val ord: Ordering[NNKey] = Ordering.by(NNKey.unapply) val entityEmbeddings = searchSpaceEmbeddings.map { embedding: EmbeddingWithEntity[B] => val entityEmbedding = EntityEmbedding(embedding.entityId, Embedding(embedding.embedding.toArray)) entityEmbedding } val shardedSearchSpace = shard(entityEmbeddings, searchSpaceShards) val groupedSearchSpaceEmbeddings = shardedSearchSpace .flatMap { entityEmbedding => val searchGroup = Random.nextInt(numOfSearchGroups) (0 until numReplicas).map { replica => (NNKey(searchGroup, replica, Some(numReplicas)), entityEmbedding) } } val shardedQueries = shard(queryEmbeddings, queryShards) val groupedQueryEmbeddings = shardedQueries .flatMap { entity => val replica = Random.nextInt(numReplicas) (0 until numOfSearchGroups).map { searchGroup => (NNKey(searchGroup, replica, Some(numReplicas)), entity) } }.group .withReducers(reducersOption.getOrElse(numOfSearchGroups * numReplicas)) val numberAnnIndexQueries = Stat("NumberAnnIndexQueries") val annIndexQueryTotalMs = Stat("AnnIndexQueryTotalMs") val numberIndexBuilds = Stat("NumberIndexBuilds") val annIndexBuildTotalMs = Stat("AnnIndexBuildTotalMs") val groupedKnn = groupedQueryEmbeddings .cogroup(groupedSearchSpaceEmbeddings) { case (_, queryIter, searchSpaceIter) => // This index build happens numReplicas times. Ideally we could serialize the queryable. // And only build the index once per search group. // The issues with that now are: // - The HNSW queryable is not serializable in scalding // - The way that map reduce works requires that there is a job that write out the search // space embeddings numReplicas times. In the current setup, we can do that by sharding // the embeddings first and then fanning out. But if we had a single queryable, we would // not be able to shard it easily and writing this out would take a long time. val indexBuildStartTime = System.currentTimeMillis() val queryable = indexingStrategy.buildIndex(searchSpaceIter) if (useCounters) { numberIndexBuilds.inc() annIndexBuildTotalMs.incBy(System.currentTimeMillis() - indexBuildStartTime) } queryIter.flatMap { query => val queryStartTime = System.currentTimeMillis() val embedding = Embedding(query.embedding.toArray) val result = Await.result( queryable.queryWithDistance(embedding, numNeighbors) ) val queryToTopNeighbors = result .map { neighbor => (query.entityId, (neighbor.neighbor, neighbor.distance)) } if (useCounters) { numberAnnIndexQueries.inc() annIndexQueryTotalMs.incBy(System.currentTimeMillis() - queryStartTime) } queryToTopNeighbors } } .values .group val groupedKnnWithReducers = reducersOption .map { reducers => groupedKnn .withReducers(reducers) }.getOrElse(groupedKnn) groupedKnnWithReducers .sortedTake(numNeighbors) { Ordering .by[(B, D), D] { case (_, distance) => distance } } } private[this] def shard[T]( pipe: TypedPipe[T], numberOfShards: Option[Int] ): TypedPipe[T] = { numberOfShards .map { shards => pipe.shard(shards) }.getOrElse(pipe) } private[this] def findNearestNeighboursViaCross[A <: EntityId, B <: EntityId, D <: Distance[D]]( queryEmbeddings: TypedPipe[EmbeddingWithEntity[A]], searchSpaceEmbeddings: TypedPipe[EmbeddingWithEntity[B]], metric: Metric[D], numNeighbors: Int, reducers: Int, mappers: Int, isSearchSpaceLarger: Boolean )( implicit ordering: Ordering[A] ): TypedPipe[(A, Seq[(B, D)])] = { val crossed: TypedPipe[(A, (B, D))] = if (isSearchSpaceLarger) { searchSpaceEmbeddings .shard(mappers) .cross(queryEmbeddings).map { case (searchSpaceEmbedding, queryEmbedding) => val distance = metric.distance(searchSpaceEmbedding.embedding, queryEmbedding.embedding) (queryEmbedding.entityId, (searchSpaceEmbedding.entityId, distance)) } } else { queryEmbeddings .shard(mappers) .cross(searchSpaceEmbeddings).map { case (queryEmbedding, searchSpaceEmbedding) => val distance = metric.distance(searchSpaceEmbedding.embedding, queryEmbedding.embedding) (queryEmbedding.entityId, (searchSpaceEmbedding.entityId, distance)) } } crossed .groupBy(_._1) .withReducers(reducers) .sortedTake(numNeighbors) { Ordering .by[(A, (B, D)), D] { case (_, (_, distance)) => distance } // Sort by distance metric in ascending order }.map { case (queryId, neighbors) => (queryId, neighbors.map(_._2)) } } /** * Convert nearest neighbors to string format. * By default format would be (queryId neighbourId:distance neighbourId:distance .....) in ascending order of distance. * @param nearestNeighbors nearest neighbors tuple in form of (queryId, Seq[(neighborId, distance)] * @param queryEntityKind entity kind of query * @param neighborEntityKind entity kind of search space/neighbors * @param idDistanceSeparator String separator to separate a single neighborId and distance. Default to colon (:) * @param neighborSeparator String operator to separate neighbors. Default to tab * @tparam A type of query entity * @tparam B type of search space entity * @tparam D type of distance */ def nearestNeighborsToString[A <: EntityId, B <: EntityId, D <: Distance[D]]( nearestNeighbors: (A, Seq[(B, D)]), queryEntityKind: EntityKind[A], neighborEntityKind: EntityKind[B], idDistanceSeparator: String = ":", neighborSeparator: String = "\t" ): String = { val (queryId, neighbors) = nearestNeighbors val formattedNeighbors = neighbors.map { case (neighbourId, distance) => s"${neighborEntityKind.stringInjection.apply(neighbourId)}$idDistanceSeparator${distance.distance}" } (queryEntityKind.stringInjection.apply(queryId) +: formattedNeighbors) .mkString(neighborSeparator) } private[this] case class NNKey( searchGroup: Int, replica: Int, maxReplica: Option[Int] = None) { override def hashCode(): Int = maxReplica.map(_ * searchGroup + replica).getOrElse(super.hashCode()) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/KnnOfflineJob.scala ================================================ package com.twitter.ann.scalding.offline import com.twitter.ann.common.Metric import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.ml.featurestore.lib.UserId import com.twitter.ml.featurestore.lib.embedding.EmbeddingWithEntity import com.twitter.cortex.ml.embeddings.common.EntityKind import com.twitter.entityembeddings.neighbors.thriftscala.{EntityKey, NearestNeighbors} import com.twitter.scalding.commons.source.VersionedKeyValSource import com.twitter.scalding.typed.TypedPipe import com.twitter.scalding.{Args, DateOps, DateParser, DateRange, Execution, TypedTsv, UniqueID} import com.twitter.scalding_internal.job.TwitterExecutionApp import com.twitter.search.common.file.{AbstractFile, LocalFile} import java.util.TimeZone /** * Generates the nearest neighbour for users and store them in Manhattan format i.e sequence files. * See README for oscar usage. */ object KnnOfflineJob extends TwitterExecutionApp { override def job: Execution[Unit] = Execution.withId { implicit uniqueId => Execution.getArgs.flatMap { args: Args => val knnDirectoryOpt: Option[String] = args.optional("knn_directory") knnDirectoryOpt match { case Some(knnDirectory) => Execution.withCachedFile(knnDirectory) { directory => execute(args, Some(new LocalFile(directory.file))) } case None => execute(args, None) } } } /** * Execute KnnOfflineJob * @param args: The args object for this job * @param abstractFile: An optional of producer embedding path */ def execute( args: Args, abstractFile: Option[AbstractFile] )( implicit uniqueID: UniqueID ): Execution[Unit] = { implicit val tz: TimeZone = TimeZone.getDefault() implicit val dp: DateParser = DateParser.default implicit val dateRange = DateRange.parse(args.list("date"))(DateOps.UTC, DateParser.default) implicit val keyInject = BinaryScalaCodec(EntityKey) implicit val valueInject = BinaryScalaCodec(NearestNeighbors) val entityKind = EntityKind.getEntityKind(args("producer_entity_kind")) val metric = Metric.fromString(args("metric")) val outputPath: String = args("output_path") val numNeighbors: Int = args("neighbors").toInt val ef = args.getOrElse("ef", numNeighbors.toString).toInt val reducers: Int = args("reducers").toInt val knnDimension: Int = args("dimension").toInt val debugOutputPath: Option[String] = args.optional("debug_output_path") val filterPath: Option[String] = args.optional("users_filter_path") val shards: Int = args.getOrElse("shards", "100").toInt val useHashJoin: Boolean = args.getOrElse("use_hash_join", "false").toBoolean val mhOutput = VersionedKeyValSource[EntityKey, NearestNeighbors]( path = outputPath, sourceVersion = None, sinkVersion = None, maxFailures = 0, versionsToKeep = 1 ) val consumerEmbeddings: TypedPipe[EmbeddingWithEntity[UserId]] = KnnHelper.getFilteredUserEmbeddings( args, filterPath, reducers, useHashJoin ) val neighborsPipe: TypedPipe[(EntityKey, NearestNeighbors)] = KnnHelper.getNeighborsPipe( args, entityKind, metric, ef, consumerEmbeddings, abstractFile, reducers, numNeighbors, knnDimension ) val neighborsExecution: Execution[Unit] = neighborsPipe .writeExecution(mhOutput) // Write manual Inspection debugOutputPath match { case Some(path: String) => val debugExecution: Execution[Unit] = KnnDebug .getDebugTable( neighborsPipe = neighborsPipe, shards = shards, reducers = reducers ) .writeExecution(TypedTsv(path)) Execution.zip(debugExecution, neighborsExecution).unit case None => neighborsExecution } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/KnnTruthSetGenerator.scala ================================================ package com.twitter.ann.scalding.offline import com.twitter.ann.common.Distance import com.twitter.ann.common.Metric import com.twitter.ann.scalding.offline.KnnHelper.nearestNeighborsToString import com.twitter.cortex.ml.embeddings.common.EntityKind import com.twitter.ml.featurestore.lib.EntityId import com.twitter.scalding.source.TypedText import com.twitter.scalding.Args import com.twitter.scalding.Execution import com.twitter.scalding.UniqueID import com.twitter.scalding_internal.job.TwitterExecutionApp /** * This job reads index embedding data, query embeddings data, and split into index set, query set and true nearest neigbor set * from query to index. */ object KnnTruthSetGenerator extends TwitterExecutionApp { override def job: Execution[Unit] = Execution.withId { implicit uniqueId => Execution.getArgs.flatMap { args: Args => val queryEntityKind = EntityKind.getEntityKind(args("query_entity_kind")) val indexEntityKind = EntityKind.getEntityKind(args("index_entity_kind")) val metric = Metric.fromString(args("metric")) run(queryEntityKind, indexEntityKind, metric, args) } } private[this] def run[A <: EntityId, B <: EntityId, D <: Distance[D]]( uncastQueryEntityKind: EntityKind[_], uncastIndexSpaceEntityKind: EntityKind[_], uncastMetric: Metric[_], args: Args )( implicit uniqueID: UniqueID ): Execution[Unit] = { val queryEntityKind = uncastQueryEntityKind.asInstanceOf[EntityKind[A]] val indexEntityKind = uncastIndexSpaceEntityKind.asInstanceOf[EntityKind[B]] val metric = uncastMetric.asInstanceOf[Metric[D]] val reducers = args.int("reducers") val mappers = args.int("mappers") val numNeighbors = args.int("neighbors") val knnOutputPath = args("truth_set_output_path") val querySamplePercent = args.double("query_sample_percent", 100) / 100 val indexSamplePercent = args.double("index_sample_percent", 100) / 100 val queryEmbeddings = queryEntityKind.parser .getEmbeddingFormat(args, "query") .getEmbeddings .sample(querySamplePercent) val indexEmbeddings = indexEntityKind.parser .getEmbeddingFormat(args, "index") .getEmbeddings .sample(indexSamplePercent) // calculate and write knn val knnExecution = KnnHelper .findNearestNeighbours( queryEmbeddings, indexEmbeddings, metric, numNeighbors, reducers = reducers, mappers = mappers )(queryEntityKind.ordering, uniqueID).map( nearestNeighborsToString(_, queryEntityKind, indexEntityKind) ) .shard(1) .writeExecution(TypedText.tsv(knnOutputPath)) // write query set embeddings val querySetExecution = queryEntityKind.parser .getEmbeddingFormat(args, "query_set_output") .writeEmbeddings(queryEmbeddings) // write index set embeddings val indexSetExecution = indexEntityKind.parser .getEmbeddingFormat(args, "index_set_output") .writeEmbeddings(indexEmbeddings) Execution.zip(knnExecution, querySetExecution, indexSetExecution).unit } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/ParameterlessQueryable.scala ================================================ package com.twitter.ann.scalding.offline import com.twitter.ann.common.EmbeddingType.EmbeddingVector import com.twitter.ann.common.{Distance, NeighborWithDistance, Queryable, RuntimeParams} import com.twitter.util.Future private[offline] case class ParameterlessQueryable[T, P <: RuntimeParams, D <: Distance[D]]( queryable: Queryable[T, P, D], runtimeParamsForAllQueries: P) { /** * ANN query for ids with distance. * * @param embedding : Embedding/Vector to be queried with. * @param numOfNeighbors : Number of neighbours to be queried for. * * @return List of approximate nearest neighbour ids with distance from the query embedding. */ def queryWithDistance( embedding: EmbeddingVector, numOfNeighbors: Int ): Future[List[NeighborWithDistance[T, D]]] = queryable.queryWithDistance(embedding, numOfNeighbors, runtimeParamsForAllQueries) } ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/README ================================================ # Description This pipeline uses hnsw and scalding to create an hnsw index based on producers embeddings, which it then uses to construct lists of producer suggestions for each user. ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/faissindexbuilder/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java11", tags = [ "bazel-compatible", "bazel-only", ], dependencies = [ "3rdparty/src/jvm/com/twitter/scalding:args", "3rdparty/src/jvm/com/twitter/scalding:core", "ann/src/main/scala/com/twitter/ann/annoy", "ann/src/main/scala/com/twitter/ann/brute_force", "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/scala/com/twitter/ann/faiss", "ann/src/main/scala/com/twitter/ann/serialization", "ann/src/main/scala/com/twitter/ann/util", "src/scala/com/twitter/cortex/ml/embeddings/common:Helpers", "src/scala/com/twitter/scalding_internal/job", ], ) hadoop_binary( name = "faissindexbuilder-deploy", main = "com.twitter.ann.scalding.offline.faissindexbuilder.IndexBuilderApp", platform = "java11", runtime_platform = "java11", tags = [ "bazel-compatible", "bazel-compatible:migrated", "bazel-only", ], dependencies = [ ":faissindexbuilder", "3rdparty/jvm/org/slf4j:slf4j-jdk14", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/faissindexbuilder/IndexBuilder.scala ================================================ package com.twitter.ann.scalding.offline.faissindexbuilder import com.twitter.ann.common.Distance import com.twitter.ann.common.EntityEmbedding import com.twitter.ann.common.Metric import com.twitter.ann.faiss.FaissIndexer import com.twitter.cortex.ml.embeddings.common.EmbeddingFormat import com.twitter.ml.api.embedding.Embedding import com.twitter.ml.featurestore.lib.UserId import com.twitter.scalding.Execution import com.twitter.search.common.file.AbstractFile import com.twitter.util.logging.Logging object IndexBuilder extends FaissIndexer with Logging { def run[T <: UserId, D <: Distance[D]]( embeddingFormat: EmbeddingFormat[T], embeddingLimit: Option[Int], sampleRate: Float, factoryString: String, metric: Metric[D], outputDirectory: AbstractFile, numDimensions: Int ): Execution[Unit] = { val embeddingsPipe = embeddingFormat.getEmbeddings val limitedEmbeddingsPipe = embeddingLimit .map { limit => embeddingsPipe.limit(limit) }.getOrElse(embeddingsPipe) val annEmbeddingPipe = limitedEmbeddingsPipe.map { embedding => val embeddingSize = embedding.embedding.length assert( embeddingSize == numDimensions, s"Specified number of dimensions $numDimensions does not match the dimensions of the " + s"embedding $embeddingSize" ) EntityEmbedding[Long](embedding.entityId.userId, Embedding(embedding.embedding.toArray)) } build(annEmbeddingPipe, sampleRate, factoryString, metric, outputDirectory) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/faissindexbuilder/IndexBuilderApp.scala ================================================ package com.twitter.ann.scalding.offline.faissindexbuilder import com.twitter.ann.common.Distance import com.twitter.ann.common.Metric import com.twitter.cortex.ml.embeddings.common._ import com.twitter.ml.featurestore.lib.UserId import com.twitter.scalding.Args import com.twitter.scalding.DateOps import com.twitter.scalding.DateParser import com.twitter.scalding.DateRange import com.twitter.scalding.Execution import com.twitter.scalding_internal.job.TwitterExecutionApp import com.twitter.search.common.file.FileUtils import com.twitter.util.logging.Logging import java.util.Calendar import java.util.TimeZone trait IndexBuilderExecutable extends Logging { // This method is used to cast the entityKind and the metric to have parameters. def indexBuilderExecution[T <: UserId, D <: Distance[D]]( args: Args ): Execution[Unit] = { // parse the arguments for this job val uncastEntityKind = EntityKind.getEntityKind(args("entity_kind")) val uncastMetric = Metric.fromString(args("metric")) val entityKind = uncastEntityKind.asInstanceOf[EntityKind[T]] val metric = uncastMetric.asInstanceOf[Metric[D]] val uncastDateRange = args.list("embedding_date_range") val embeddingDateRange = if (uncastDateRange.nonEmpty) { Some(DateRange.parse(uncastDateRange)(DateOps.UTC, DateParser.default)) } else { None } val embeddingFormat = entityKind.parser.getEmbeddingFormat(args, "input", providedDateRange = embeddingDateRange) val numDimensions = args.int("num_dimensions") val embeddingLimit = args.optional("embedding_limit").map(_.toInt) val outputDirectory = FileUtils.getFileHandle(args("output_dir")) val factoryString = args.optional("factory_string").get val sampleRate = args.float("training_sample_rate", 0.05f) logger.debug(s"Job args: ${args.toString}") val finalOutputDirectory = embeddingDateRange .map { range => val cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")) cal.setTime(range.end) outputDirectory .getChild(s"${cal.get(Calendar.YEAR)}") .getChild(f"${cal.get(Calendar.MONTH) + 1}%02d") .getChild(f"${cal.get(Calendar.DAY_OF_MONTH)}%02d") }.getOrElse(outputDirectory) logger.info(s"Final output directory is ${finalOutputDirectory.getPath}") IndexBuilder .run( embeddingFormat, embeddingLimit, sampleRate, factoryString, metric, finalOutputDirectory, numDimensions ).onComplete { _ => Unit } } } object IndexBuilderApp extends TwitterExecutionApp with IndexBuilderExecutable { override def job: Execution[Unit] = Execution.getArgs.flatMap { args: Args => indexBuilderExecution(args) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/indexbuilder/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = [ "bazel-compatible", "bazel-only", ], dependencies = [ "3rdparty/src/jvm/com/twitter/scalding:args", "3rdparty/src/jvm/com/twitter/scalding:core", "ann/src/main/scala/com/twitter/ann/annoy", "ann/src/main/scala/com/twitter/ann/brute_force", "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/scala/com/twitter/ann/hnsw", "ann/src/main/scala/com/twitter/ann/serialization", "ann/src/main/scala/com/twitter/ann/util", "src/scala/com/twitter/cortex/ml/embeddings/common:Helpers", "src/scala/com/twitter/scalding_internal/job", ], ) hadoop_binary( name = "indexbuilder-deploy", main = "com.twitter.ann.scalding.offline.indexbuilder.IndexBuilderApp", platform = "java8", runtime_platform = "java8", tags = [ "bazel-compatible", "bazel-compatible:migrated", "bazel-only", ], dependencies = [ ":indexbuilder", "3rdparty/jvm/org/slf4j:slf4j-jdk14", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/indexbuilder/IndexBuilder.scala ================================================ package com.twitter.ann.scalding.offline.indexbuilder import com.twitter.ann.common.Appendable import com.twitter.ann.common.Distance import com.twitter.ann.common.EntityEmbedding import com.twitter.ann.common.Serialization import com.twitter.ann.util.IndexBuilderUtils import com.twitter.cortex.ml.embeddings.common.EmbeddingFormat import com.twitter.ml.api.embedding.Embedding import com.twitter.ml.featurestore.lib.EntityId import com.twitter.scalding.Execution import com.twitter.scalding_internal.job.FutureHelper import com.twitter.search.common.file.AbstractFile import com.twitter.util.logging.Logger object IndexBuilder { private[this] val Log = Logger.apply[IndexBuilder.type] def run[T <: EntityId, _, D <: Distance[D]]( embeddingFormat: EmbeddingFormat[T], embeddingLimit: Option[Int], index: Appendable[T, _, D] with Serialization, concurrencyLevel: Int, outputDirectory: AbstractFile, numDimensions: Int ): Execution[Unit] = { val embeddingsPipe = embeddingFormat.getEmbeddings val limitedEmbeddingsPipe = embeddingLimit .map { limit => embeddingsPipe.limit(limit) }.getOrElse(embeddingsPipe) val annEmbeddingPipe = limitedEmbeddingsPipe.map { embedding => val embeddingSize = embedding.embedding.length assert( embeddingSize == numDimensions, s"Specified number of dimensions $numDimensions does not match the dimensions of the " + s"embedding $embeddingSize" ) EntityEmbedding[T](embedding.entityId, Embedding(embedding.embedding.toArray)) } annEmbeddingPipe.toIterableExecution.flatMap { annEmbeddings => val future = IndexBuilderUtils.addToIndex(index, annEmbeddings.toStream, concurrencyLevel) val result = future.map { numberUpdates => Log.info(s"Performed $numberUpdates updates") index.toDirectory(outputDirectory) Log.info(s"Finished writing to $outputDirectory") } FutureHelper.executionFrom(result).unit } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/indexbuilder/IndexBuilderApp.scala ================================================ package com.twitter.ann.scalding.offline.indexbuilder import com.twitter.ann.annoy.TypedAnnoyIndex import com.twitter.ann.brute_force.SerializableBruteForceIndex import com.twitter.ann.common.Distance import com.twitter.ann.common.Metric import com.twitter.ann.common.ReadWriteFuturePool import com.twitter.ann.hnsw.TypedHnswIndex import com.twitter.ann.serialization.thriftscala.PersistedEmbedding import com.twitter.ann.serialization.PersistedEmbeddingInjection import com.twitter.ann.serialization.ThriftIteratorIO import com.twitter.cortex.ml.embeddings.common._ import com.twitter.ml.featurestore.lib.EntityId import com.twitter.scalding.Args import com.twitter.scalding.Execution import com.twitter.scalding_internal.job.TwitterExecutionApp import com.twitter.search.common.file.FileUtils import com.twitter.util.FuturePool import java.util.concurrent.Executors trait IndexBuilderExecutable { // This method is used to cast the entityKind and the metric to have parameters. def indexBuilderExecution[T <: EntityId, D <: Distance[D]]( args: Args ): Execution[Unit] = { // parse the arguments for this job val uncastEntityKind = EntityKind.getEntityKind(args("entity_kind")) val uncastMetric = Metric.fromString(args("metric")) val entityKind = uncastEntityKind.asInstanceOf[EntityKind[T]] val metric = uncastMetric.asInstanceOf[Metric[D]] val embeddingFormat = entityKind.parser.getEmbeddingFormat(args, "input") val injection = entityKind.byteInjection val numDimensions = args.int("num_dimensions") val embeddingLimit = args.optional("embedding_limit").map(_.toInt) val concurrencyLevel = args.int("concurrency_level") val outputDirectory = FileUtils.getFileHandle(args("output_dir")) println(s"Job args: ${args.toString}") val threadPool = Executors.newFixedThreadPool(concurrencyLevel) val serialization = args("algo") match { case "brute_force" => val PersistedEmbeddingIO = new ThriftIteratorIO[PersistedEmbedding](PersistedEmbedding) SerializableBruteForceIndex[T, D]( metric, FuturePool.apply(threadPool), new PersistedEmbeddingInjection[T](injection), PersistedEmbeddingIO ) case "annoy" => TypedAnnoyIndex.indexBuilder[T, D]( numDimensions, args.int("annoy_num_trees"), metric, injection, FuturePool.apply(threadPool) ) case "hnsw" => val efConstruction = args.int("ef_construction") val maxM = args.int("max_m") val expectedElements = args.int("expected_elements") TypedHnswIndex.serializableIndex[T, D]( numDimensions, metric, efConstruction, maxM, expectedElements, injection, ReadWriteFuturePool(FuturePool.apply(threadPool)) ) } IndexBuilder .run( embeddingFormat, embeddingLimit, serialization, concurrencyLevel, outputDirectory, numDimensions ).onComplete { _ => threadPool.shutdown() Unit } } } object IndexBuilderApp extends TwitterExecutionApp with IndexBuilderExecutable { override def job: Execution[Unit] = Execution.getArgs.flatMap { args: Args => indexBuilderExecution(args) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/indexbuilder/README.rst ================================================ ******** Overview ******** This job reads embedding data from HDFS in the embedding formats supported by the cortex MLX team. It converts that data into the ANN format and adds it to an ANN index. The ANN index is serialized and save to disk. ***************** Running In Aurora ***************** Set up example ============== This job builds an ANN index based on hnsw algorithm using user embeddings available in hdfs. .. code-block:: bash $ export JOB_NAME=ann_index_builder $ export OUTPUT_PATH=hdfs:///user/$USER/${JOB_NAME}_test $ CPU=32 RAM_GB=150 DISK_GB=60 aurora job create smf1/$USER/devel/$JOB_NAME ann/src/main/aurora/index_builder/aurora_builder.aurora \ --bind=profile.name=$JOB_NAME \ --bind=profile.role=$USER \ --bind=profile.output_dir=$OUTPUT_PATH \ --bind=profile.entity_kind=user \ --bind=profile.embedding_args='--input.embedding_format tab --input.embedding_path /user/cortex-mlx/official_examples/ann/non_pii_random_user_embeddings_tab_format' \ --bind=profile.num_dimensions=300 \ --bind=profile.algo=hnsw \ --bind=profile.ef_construction=200 \ --bind=profile.max_m=16 \ --bind=profile.expected_elements=10000000 \ --bind=profile.metric=InnerProduct \ --bind=profile.concurrency_level=32 \ --bind=profile.hadoop_cluster=dw2-smf1 This job builds an ANN index based on hnsw algorithm using producer embeddings (Major version 1546473691) available in feature store. .. code-block:: bash $ export JOB_NAME=ann_index_builder $ export OUTPUT_PATH=hdfs:///user/$USER/${JOB_NAME}_test $ CPU=32 RAM_GB=150 DISK_GB=60 aurora job create smf1/$USER/devel/$JOB_NAME ann/src/main/aurora/index_builder/aurora_builder.aurora \ --bind=profile.name=$JOB_NAME \ --bind=profile.role=$USER \ --bind=profile.output_dir=$OUTPUT_PATH \ --bind=profile.entity_kind=user \ --bind=profile.embedding_args='--input.feature_store_embedding ProducerFollowEmbedding300Dataset --input.feature_store_major_version 1546473691 --input.date_range 2019-01-02' \ --bind=profile.num_dimensions=300 \ --bind=profile.algo=hnsw \ --bind=profile.ef_construction=200 \ --bind=profile.max_m=16 \ --bind=profile.expected_elements=10000000 \ --bind=profile.metric=InnerProduct \ --bind=profile.concurrency_level=32 \ --bind=profile.hadoop_cluster=dw2-smf1 ************* Job arguments ************* Enviroment variables (resources): ============== - **CPU** Number of cpu cores (default: 32) - **RAM_GB** RAM in gigabytes (default: 150) - **DISK_GB** Disk in gigabytes (default: 60) General arguments (specified as **--profile.{options}**): ============== - **name** Aurora job name - **role** Aurora role - **hadoop_cluster** Hadoop cluster for data. dw2-smf1/proc-atla. - **input_dir** Path of saved embeddings in hdfs without prefixing `hdfs://` - **entity_kind** The type of entity id that is use with the embeddings. Possible options: - word - url - user - tweet - tfwId - **embedding_args** Embedding format args. See the documentation in `com.twitter.cortex.ml.embeddings.common.EmbeddingFormatArgsParser` for a full explanation of the input options. Possible options: 1. **input.embedding_format** Format of the serialized embedding. - usertensor - usercontinuous - comma - tab 2. **input.embedding_path** Path of saved embeddings in hdfs without prefixing `hdfs://` 3. **input.{feature_store_args}** For feature store related args like `feature_store_embedding`, `feature_store_major_version`, `date_range`: - **output_dir** Where to save the produced serialized ann index. Save to HDFS by specifying the full URI. e.g `hdfs://hadoop-dw2-nn.smf1.twitter.com/user//index_file` or using the default cluster `hdfs:///user//index_file`. - **num_dimensions** Dimension of embedding in the input data. An exception will be thrown if any entry does not have a number of dimensions equal to this number. - **metric** Distance metric (InnerProduct/Cosine/L2) - **concurrency_level** Specifies how many parallel inserts happen to the index. This should probably be set to the number of cores on the machine. - **algo** The kind of index you want to ouput. The supported options right now are: 1. **hnsw** (Metric supported: Cosine, L2, InnerProduct) .. _hnsw: https://arxiv.org/abs/1603.09320 - **ef\_construction** : Larger value increases build time but will give better recall. Good start value : 200 - **max\_m** : Larger value increases will increase the index size but will give better recall. Optimal Range : 6-48. Good starting value 16. - **expected\_elements** : Approximate number of elements that will be indexed. 2. **annoy** (Metric supported: Cosine, L2) .. _annoy: https://github.com/spotify/annoy - **annoy\_num\_trees** This parameter is required for annoy. From the annoy documentation: num_trees is provided during build time and affects the build time and the index size. A larger value will give more accurate results, but larger indexes. 3. **brute_force** (Metric supported: Cosine, L2, InnerProduct) Developing locally =================== For building and testing custom ann index builder job, You can create job bundle locally, upload to packer and then it can be used with the job using `profile.packer_package` for name, `profile.packer_role` for role and `profile.packer_version` for bundle version. .. code-block:: bash ./bazel bundle ann/src/main/scala/com/twitter/ann/scalding/offline/indexbuilder:indexbuilder-deploy \ --bundle-jvm-archive=zip .. code-block:: bash packer add_version --cluster=atla dist/indexbuilder-deploy.zip ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/indexbuilderfrombq/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = [ "bazel-compatible", "bazel-only", ], dependencies = [ "3rdparty/src/jvm/com/twitter/scalding:args", "3rdparty/src/jvm/com/twitter/scalding:core", "ann/src/main/scala/com/twitter/ann/annoy", "ann/src/main/scala/com/twitter/ann/brute_force", "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/scala/com/twitter/ann/hnsw", "ann/src/main/scala/com/twitter/ann/serialization", "ann/src/main/scala/com/twitter/ann/util", "src/scala/com/twitter/cortex/ml/embeddings/common:Helpers", "src/scala/com/twitter/scalding_internal/bigquery", "src/scala/com/twitter/scalding_internal/job", ], ) hadoop_binary( name = "ann-index-builder", main = "com.twitter.ann.scalding.offline.indexbuilderfrombq.IndexBuilderFromBQApp", platform = "java8", runtime_platform = "java8", tags = [ "bazel-compatible", "bazel-compatible:migrated", "bazel-only", ], dependencies = [ ":indexbuilderfrombq", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/indexbuilderfrombq/IndexBuilderFromBQ.scala ================================================ package com.twitter.ann.scalding.offline.indexbuilderfrombq import com.twitter.ann.common.Appendable import com.twitter.ann.common.Distance import com.twitter.ann.common.EntityEmbedding import com.twitter.ann.common.Serialization import com.twitter.ann.util.IndexBuilderUtils import com.twitter.ml.api.embedding.Embedding import com.twitter.ml.featurestore.lib.embedding.EmbeddingWithEntity import com.twitter.ml.featurestore.lib.EntityId import com.twitter.scalding.Execution import com.twitter.scalding.TypedPipe import com.twitter.scalding_internal.job.FutureHelper import com.twitter.search.common.file.AbstractFile import com.twitter.util.logging.Logger object IndexBuilder { private[this] val Log = Logger.apply[IndexBuilder.type] def run[T <: EntityId, _, D <: Distance[D]]( embeddingsPipe: TypedPipe[EmbeddingWithEntity[T]], embeddingLimit: Option[Int], index: Appendable[T, _, D] with Serialization, concurrencyLevel: Int, outputDirectory: AbstractFile, numDimensions: Int ): Execution[Unit] = { val limitedEmbeddingsPipe = embeddingLimit .map { limit => embeddingsPipe.limit(limit) }.getOrElse(embeddingsPipe) val annEmbeddingPipe = limitedEmbeddingsPipe.map { embedding => val embeddingSize = embedding.embedding.length assert( embeddingSize == numDimensions, s"Specified number of dimensions $numDimensions does not match the dimensions of the " + s"embedding $embeddingSize" ) EntityEmbedding[T](embedding.entityId, Embedding(embedding.embedding.toArray)) } annEmbeddingPipe.toIterableExecution.flatMap { annEmbeddings => val future = IndexBuilderUtils.addToIndex(index, annEmbeddings.toStream, concurrencyLevel) val result = future.map { numberUpdates => Log.info(s"Performed $numberUpdates updates") index.toDirectory(outputDirectory) Log.info(s"Finished writing to $outputDirectory") } FutureHelper.executionFrom(result).unit } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/scalding/offline/indexbuilderfrombq/IndexBuilderFromBQApp.scala ================================================ package com.twitter.ann.scalding.offline.indexbuilderfrombq import com.google.auth.oauth2.ServiceAccountCredentials import com.google.cloud.bigquery.BigQueryOptions import com.google.cloud.bigquery.QueryJobConfiguration import com.twitter.ann.annoy.TypedAnnoyIndex import com.twitter.ann.brute_force.SerializableBruteForceIndex import com.twitter.ann.common.Distance import com.twitter.ann.common.Metric import com.twitter.ann.common.ReadWriteFuturePool import com.twitter.ann.hnsw.TypedHnswIndex import com.twitter.ann.serialization.PersistedEmbeddingInjection import com.twitter.ann.serialization.ThriftIteratorIO import com.twitter.ann.serialization.thriftscala.PersistedEmbedding import com.twitter.cortex.ml.embeddings.common._ import com.twitter.ml.api.embedding.Embedding import com.twitter.ml.featurestore.lib._ import com.twitter.ml.featurestore.lib.embedding.EmbeddingWithEntity import com.twitter.scalding.Args import com.twitter.scalding.Execution import com.twitter.scalding.typed.TypedPipe import com.twitter.scalding_internal.bigquery.BigQueryConfig import com.twitter.scalding_internal.bigquery.BigQuerySource import com.twitter.scalding_internal.job.TwitterExecutionApp import com.twitter.scalding_internal.multiformat.format.keyval.KeyVal import com.twitter.search.common.file.FileUtils import com.twitter.util.FuturePool import java.io.FileInputStream import java.time.LocalDateTime import java.time.ZoneOffset import java.util.concurrent.Executors import org.apache.avro.generic.GenericRecord import scala.collection.JavaConverters._ /** * Scalding execution app for building ANN index from embeddings present in BigQuery table. * The output index is written to a GCS file. * * Note: * - Assumes input data has the fields entityId * - Assumes input data has the fields embedding * * Command for running the app (from source repo root): * scalding remote run \ * --target ann/src/main/scala/com/twitter/ann/scalding/offline/indexbuilderfrombq:ann-index-builder-binary */ trait IndexBuilderFromBQExecutable { // This method is used to cast the entityKind and the metric to have parameters. def indexBuilderExecution[T <: EntityId, D <: Distance[D]]( args: Args ): Execution[Unit] = { // parse the arguments for this job val uncastEntityKind = EntityKind.getEntityKind(args("entity_kind")) val uncastMetric = Metric.fromString(args("metric")) val entityKind = uncastEntityKind.asInstanceOf[EntityKind[T]] val metric = uncastMetric.asInstanceOf[Metric[D]] val injection = entityKind.byteInjection val numDimensions = args.int("num_dimensions") val embeddingLimit = args.optional("embedding_limit").map(_.toInt) val concurrencyLevel = args.int("concurrency_level") val bigQuery = BigQueryOptions .newBuilder().setProjectId(args.required("bq_gcp_job_project")).setCredentials( ServiceAccountCredentials.fromStream( new FileInputStream(args.required("gcp_service_account_key_json")))).build().getService // Query to get the latest partition of the BigQuery table. val query = s"SELECT MAX(ts) AS RecentPartition FROM ${args.required("bq_gcp_table_project")}.${args .required("bq_dataset")}.${args.required("bq_table")}" val queryConfig = QueryJobConfiguration .newBuilder(query) .setUseLegacySql(false) .build val recentPartition = bigQuery .query(queryConfig).iterateAll().asScala.map(field => { field.get(0).getStringValue }).toArray.apply(0) // Query to extract the embeddings from the latest partition of the BigQuery table val bigQueryConfig = BigQueryConfig( args.required("bq_gcp_table_project"), args .required("bq_dataset"), args.required("bq_table")) .withServiceAccountKey(args.required("gcp_service_account_key_json")) val bqFilter = Some( s"ts >= '${recentPartition}' AND DATE(TIMESTAMP_MILLIS(createdAt)) >= DATE_SUB(DATE('${recentPartition}'), INTERVAL 1 DAY) AND DATE(TIMESTAMP_MILLIS(createdAt)) <= DATE('${recentPartition}')") val withFilterBigQueryConfig = bqFilter .map { filter: String => bigQueryConfig.withFilter(filter) }.getOrElse(bigQueryConfig) val source = new BigQuerySource(withFilterBigQueryConfig) .andThen(avroMapper) val sourcePipe = TypedPipe .from(source) .map(transform[T](entityKind)) println(s"Job args: ${args.toString}") val threadPool = Executors.newFixedThreadPool(concurrencyLevel) val serialization = args("algo") match { case "brute_force" => val PersistedEmbeddingIO = new ThriftIteratorIO[PersistedEmbedding](PersistedEmbedding) SerializableBruteForceIndex[T, D]( metric, FuturePool.apply(threadPool), new PersistedEmbeddingInjection[T](injection), PersistedEmbeddingIO ) case "annoy" => TypedAnnoyIndex.indexBuilder[T, D]( numDimensions, args.int("annoy_num_trees"), metric, injection, FuturePool.apply(threadPool) ) case "hnsw" => val efConstruction = args.int("ef_construction") val maxM = args.int("max_m") val expectedElements = args.int("expected_elements") TypedHnswIndex.serializableIndex[T, D]( numDimensions, metric, efConstruction, maxM, expectedElements, injection, ReadWriteFuturePool(FuturePool.apply(threadPool)) ) } // Output directory for the ANN index. We place the index under a timestamped directory which // will be used by the ANN service to read the latest index val timestamp = LocalDateTime.now().toEpochSecond(ZoneOffset.UTC) val outputDirectory = FileUtils.getFileHandle(args("output_dir") + "/" + timestamp) IndexBuilder .run( sourcePipe, embeddingLimit, serialization, concurrencyLevel, outputDirectory, numDimensions ).onComplete { _ => threadPool.shutdown() Unit } } def avroMapper(row: GenericRecord): KeyVal[Long, java.util.List[Double]] = { val entityId = row.get("entityId") val embedding = row.get("embedding") KeyVal( entityId.toString.toLong, embedding.asInstanceOf[java.util.List[Double]] ) } def transform[T <: EntityId]( entityKind: EntityKind[T] )( bqRecord: KeyVal[Long, java.util.List[Double]] ): EmbeddingWithEntity[T] = { val embeddingArray = bqRecord.value.asScala.map(_.floatValue()).toArray val entity_id = entityKind match { case UserKind => UserId(bqRecord.key).toThrift case TweetKind => TweetId(bqRecord.key).toThrift case TfwKind => TfwId(bqRecord.key).toThrift case SemanticCoreKind => SemanticCoreId(bqRecord.key).toThrift case _ => throw new IllegalArgumentException(s"Unsupported embedding kind: $entityKind") } EmbeddingWithEntity[T]( EntityId.fromThrift(entity_id).asInstanceOf[T], Embedding(embeddingArray)) } } /* scalding remote run \ --target ann/src/main/scala/com/twitter/ann/scalding/offline/indexbuilderfrombq:ann-index-builder-binary */ object IndexBuilderFromBQApp extends TwitterExecutionApp with IndexBuilderFromBQExecutable { override def job: Execution[Unit] = Execution.getArgs.flatMap { args: Args => indexBuilderExecution(args) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/serialization/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/bijection:core", "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/thrift/com/twitter/ann/serialization:serialization-scala", "mediaservices/commons", "scrooge/scrooge-core", "src/scala/com/twitter/scalding_internal/multiformat/format", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/serialization/DummyANNIndexInjection.scala ================================================ package com.twitter.ann.serialization import com.twitter.scalding_internal.multiformat.format.keyval.KeyValInjection import com.twitter.scalding_internal.multiformat.format.keyval.KeyValInjection.Long2BigEndian /** Dummy injection required to writeup dummy dal dataset to ANN folder. **/ object DummyANNIndexInjection { val injection: KeyValInjection[Long, Long] = KeyValInjection[Long, Long](Long2BigEndian, Long2BigEndian) } ================================================ FILE: ann/src/main/scala/com/twitter/ann/serialization/PersistedEmbeddingInjection.scala ================================================ package com.twitter.ann.serialization import com.twitter.ann.common.EntityEmbedding import com.twitter.ann.common.EmbeddingType._ import com.twitter.ann.serialization.thriftscala.PersistedEmbedding import com.twitter.bijection.Injection import com.twitter.mediaservices.commons.codec.ArrayByteBufferCodec import java.nio.ByteBuffer import scala.util.Try /** * Injection that converts from the ann.common.Embedding to the thrift PersistedEmbedding. */ class PersistedEmbeddingInjection[T]( idByteInjection: Injection[T, Array[Byte]]) extends Injection[EntityEmbedding[T], PersistedEmbedding] { override def apply(entity: EntityEmbedding[T]): PersistedEmbedding = { val byteBuffer = ByteBuffer.wrap(idByteInjection(entity.id)) PersistedEmbedding(byteBuffer, embeddingSerDe.toThrift(entity.embedding)) } override def invert(persistedEmbedding: PersistedEmbedding): Try[EntityEmbedding[T]] = { val idTry = idByteInjection.invert(ArrayByteBufferCodec.decode(persistedEmbedding.id)) idTry.map { id => EntityEmbedding(id, embeddingSerDe.fromThrift(persistedEmbedding.embedding)) } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/serialization/ThriftIteratorIO.scala ================================================ package com.twitter.ann.serialization import com.twitter.scrooge.{ThriftStruct, ThriftStructCodec} import java.io.{InputStream, OutputStream} import org.apache.thrift.protocol.TBinaryProtocol import org.apache.thrift.transport.{TIOStreamTransport, TTransportException} /** * Class that can serialize and deserialize an iterator of thrift objects. * This class can do things lazily so there is no need to have all the object into memory. */ class ThriftIteratorIO[T <: ThriftStruct]( codec: ThriftStructCodec[T]) { def toOutputStream( iterator: Iterator[T], outputStream: OutputStream ): Unit = { val protocol = (new TBinaryProtocol.Factory).getProtocol(new TIOStreamTransport(outputStream)) iterator.foreach { thriftObject => codec.encode(thriftObject, protocol) } } /** * Returns an iterator that lazily reads from an inputStream. * @return */ def fromInputStream( inputStream: InputStream ): Iterator[T] = { ThriftIteratorIO.getIterator(codec, inputStream) } } object ThriftIteratorIO { private def getIterator[T <: ThriftStruct]( codec: ThriftStructCodec[T], inputStream: InputStream ): Iterator[T] = { val protocol = (new TBinaryProtocol.Factory).getProtocol(new TIOStreamTransport(inputStream)) def getNext: Option[T] = try { Some(codec.decode(protocol)) } catch { case e: TTransportException if e.getType == TTransportException.END_OF_FILE => inputStream.close() None } Iterator .continually[Option[T]](getNext) .takeWhile(_.isDefined) // It should be safe to call get on here since we are only take the defined ones. .map(_.get) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/loadtest/AnnLoadTest.scala ================================================ package com.twitter.ann.service.loadtest import com.twitter.ann.common.EmbeddingType.EmbeddingVector import com.twitter.ann.common.{Appendable, Distance, EntityEmbedding, Queryable, RuntimeParams} import com.twitter.util.logging.Logger import com.twitter.util.{Duration, Future} class AnnIndexQueryLoadTest( worker: AnnLoadTestWorker = new AnnLoadTestWorker()) { lazy val logger = Logger(getClass.getName) def performQueries[T, P <: RuntimeParams, D <: Distance[D]]( queryable: Queryable[T, P, D], qps: Int, duration: Duration, queries: Seq[Query[T]], concurrencyLevel: Int, runtimeConfigurations: Seq[QueryTimeConfiguration[T, P]] ): Future[Unit] = { logger.info(s"Query set: ${queries.size}") val res = Future.traverseSequentially(runtimeConfigurations) { config => logger.info(s"Run load test with runtime config $config") worker.runWithQps( queryable, queries, qps, duration, config, concurrencyLevel ) } res.onSuccess { _ => logger.info(s"Done loadtest with $qps for ${duration.inMilliseconds / 1000} sec") } res.unit } } /** * @param embedding Embedding vector * @param trueNeighbours List of true neighbour ids. Empty in case true neighbours dataset not available * @tparam T Type of neighbour */ case class Query[T](embedding: EmbeddingVector, trueNeighbours: Seq[T] = Seq.empty) class AnnIndexBuildLoadTest( buildRecorder: LoadTestBuildRecorder, embeddingIndexer: EmbeddingIndexer = new EmbeddingIndexer()) { lazy val logger = Logger(getClass.getName) def indexEmbeddings[T, P <: RuntimeParams, D <: Distance[D]]( appendable: Appendable[T, P, D], indexSet: Seq[EntityEmbedding[T]], concurrencyLevel: Int ): Future[Queryable[T, P, D]] = { logger.info(s"Index set: ${indexSet.size}") val queryable = embeddingIndexer .indexEmbeddings( appendable, buildRecorder, indexSet, concurrencyLevel ).onSuccess(_ => logger.info(s"Done indexing..")) queryable } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/loadtest/AnnLoadTestMain.scala ================================================ package com.twitter.ann.service.loadtest import com.twitter.ann.annoy.AnnoyCommon import com.twitter.ann.annoy.AnnoyRuntimeParams import com.twitter.ann.annoy.TypedAnnoyIndex import com.twitter.ann.common._ import com.twitter.ann.common.thriftscala.{Distance => ServiceDistance} import com.twitter.ann.common.thriftscala.{RuntimeParams => ServiceRuntimeParams} import com.twitter.ann.faiss.FaissCommon import com.twitter.ann.faiss.FaissParams import com.twitter.ann.hnsw.HnswCommon import com.twitter.ann.hnsw.HnswParams import com.twitter.ann.hnsw.TypedHnswIndex import com.twitter.bijection.Injection import com.twitter.cortex.ml.embeddings.common.EntityKind import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.util.DefaultTimer import com.twitter.finatra.mtls.modules.ServiceIdentifierModule import com.twitter.inject.server.TwitterServer import com.twitter.util._ import java.util.concurrent.TimeUnit /** * To build and upload: * $ ./bazel bundle ann/src/main/scala/com/twitter/ann/service/loadtest:bin --bundle-jvm-archive=zip * $ packer add_version --cluster=smf1 $USER ann-loadtest dist/ann-loadtest.zip */ object AnnLoadTestMain extends TwitterServer { private[this] val algo = flag[String]("algo", "load test server types: [annoy/hnsw]") private[this] val targetQPS = flag[Int]("qps", "target QPS for load test") private[this] val queryIdType = flag[String]( "query_id_type", "query id type for load test: [long/string/int/user/tweet/word/url/tfwId]") private[this] val indexIdType = flag[String]( "index_id_type", "index id type for load test: [long/string/int/user/tweet/word/url/tfwId]") private[this] val metric = flag[String]("metric", "metric type for load test: [Cosine/L2/InnerProduct]") private[this] val durationSec = flag[Int]("duration_sec", "duration for the load test in sec") private[this] val numberOfNeighbors = flag[Seq[Int]]("number_of_neighbors", Seq(), "number of neighbors") private[this] val dimension = flag[Int]("embedding_dimension", "dimension of embeddings") private[this] val querySetDir = flag[String]("query_set_dir", "", "Directory containing the queries") private[this] val indexSetDir = flag[String]( "index_set_dir", "", "Directory containing the embeddings to be indexed" ) private[this] val truthSetDir = flag[String]("truth_set_dir", "", "Directory containing the truth data") private[this] val loadTestType = flag[String]("loadtest_type", "Load test type [server/local]") private[this] val serviceDestination = flag[String]("service_destination", "wily address of remote query service") private[this] val concurrencyLevel = flag[Int]("concurrency_level", 8, "number of concurrent operations on the index") // Queries with random embeddings private[this] val withRandomQueries = flag[Boolean]("with_random_queries", false, "query with random embeddings") private[this] val randomQueriesCount = flag[Int]("random_queries_count", 50000, "total random queries") private[this] val randomEmbeddingMinValue = flag[Float]("random_embedding_min_value", -1.0f, "Min value of random embeddings") private[this] val randomEmbeddingMaxValue = flag[Float]("random_embedding_max_value", 1.0f, "Max value of random embeddings") // parameters for annoy private[this] val numOfNodesToExplore = flag[Seq[Int]]("annoy_num_of_nodes_to_explore", Seq(), "number of nodes to explore") private[this] val numOfTrees = flag[Int]("annoy_num_trees", 0, "number of trees to build") // parameters for HNSW private[this] val efConstruction = flag[Int]("hnsw_ef_construction", "ef for Hnsw construction") private[this] val ef = flag[Seq[Int]]("hnsw_ef", Seq(), "ef for Hnsw query") private[this] val maxM = flag[Int]("hnsw_max_m", "maxM for Hnsw") // FAISS private[this] val nprobe = flag[Seq[Int]]("faiss_nprobe", Seq(), "nprobe for faiss query") private[this] val quantizerEf = flag[Seq[Int]]("faiss_quantizerEf", Seq(0), "quantizerEf for faiss query") private[this] val quantizerKfactorRF = flag[Seq[Int]]("faiss_quantizerKfactorRF", Seq(0), "quantizerEf for faiss query") private[this] val quantizerNprobe = flag[Seq[Int]]("faiss_quantizerNprobe", Seq(0), "quantizerNprobe for faiss query") private[this] val ht = flag[Seq[Int]]("faiss_ht", Seq(0), "ht for faiss query") implicit val timer: Timer = DefaultTimer override def start(): Unit = { logger.info("Starting load test..") logger.info(flag.getAll().mkString("\t")) assert(numberOfNeighbors().nonEmpty, "number_of_neighbors not defined") assert(dimension() > 0, s"Invalid dimension ${dimension()}") val inMemoryBuildRecorder = new InMemoryLoadTestBuildRecorder val queryableFuture = buildQueryable(inMemoryBuildRecorder) val queryConfig = getQueryRuntimeConfig val result = queryableFuture.flatMap { queryable => performQueries(queryable, queryConfig, getQueries) } Await.result(result) System.out.println(s"Target QPS: ${targetQPS()}") System.out.println(s"Duration per test: ${durationSec()}") System.out.println(s"Concurrency Level: ${concurrencyLevel()}") LoadTestUtils .printResults(inMemoryBuildRecorder, queryConfig) .foreach(System.out.println) Await.result(close()) System.exit(0) } private[this] def getQueries[Q, I]: Seq[Query[I]] = { if (withRandomQueries()) { assert( truthSetDir().isEmpty, "Cannot use truth set when query with random embeddings enabled" ) val queries = LoadTestUtils.getRandomQuerySet( dimension(), randomQueriesCount(), randomEmbeddingMinValue(), randomEmbeddingMaxValue() ) queries.map(Query[I](_)) } else { assert(querySetDir().nonEmpty, "Query set path is empty") assert(queryIdType().nonEmpty, "Query id type is empty") val queries = LoadTestUtils.getEmbeddingsSet[Q](querySetDir(), queryIdType()) if (truthSetDir().nonEmpty) { // Join the queries with truth set data. assert(indexIdType().nonEmpty, "Index id type is empty") val truthSetMap = LoadTestUtils.getTruthSetMap[Q, I](truthSetDir(), queryIdType(), indexIdType()) queries.map(entity => Query[I](entity.embedding, truthSetMap(entity.id))) } else { queries.map(entity => Query[I](entity.embedding)) } } } private[this] def getQueryRuntimeConfig[ T, P <: RuntimeParams ]: Seq[QueryTimeConfiguration[T, P]] = { val queryTimeConfig = algo() match { case "annoy" => assert(numOfNodesToExplore().nonEmpty, "Must specify the num_of_nodes_to_explore") logger.info(s"Querying annoy index with num_of_nodes_to_explore ${numOfNodesToExplore()}") for { numNodes <- numOfNodesToExplore() numOfNeighbors <- numberOfNeighbors() } yield { buildQueryTimeConfig[T, AnnoyRuntimeParams]( numOfNeighbors, AnnoyRuntimeParams(Some(numNodes)), Map( "numNodes" -> numNodes.toString, "numberOfNeighbors" -> numOfNeighbors.toString ) ).asInstanceOf[QueryTimeConfiguration[T, P]] } case "hnsw" => assert(ef().nonEmpty, "Must specify ef") logger.info(s"Querying hnsw index with ef ${ef()}") for { ef <- ef() numOfNeighbors <- numberOfNeighbors() } yield { buildQueryTimeConfig[T, HnswParams]( numOfNeighbors, HnswParams(ef), Map( "efConstruction" -> ef.toString, "numberOfNeighbors" -> numOfNeighbors.toString ) ).asInstanceOf[QueryTimeConfiguration[T, P]] } case "faiss" => assert(nprobe().nonEmpty, "Must specify nprobe") def toNonZeroOptional(x: Int): Option[Int] = if (x != 0) Some(x) else None for { numOfNeighbors <- numberOfNeighbors() runNProbe <- nprobe() runQEF <- quantizerEf() runKFactorEF <- quantizerKfactorRF() runQNProbe <- quantizerNprobe() runHT <- ht() } yield { val params = FaissParams( Some(runNProbe), toNonZeroOptional(runQEF), toNonZeroOptional(runKFactorEF), toNonZeroOptional(runQNProbe), toNonZeroOptional(runHT)) buildQueryTimeConfig[T, FaissParams]( numOfNeighbors, params, Map( "nprobe" -> params.nprobe.toString, "quantizer_efSearch" -> params.quantizerEf.toString, "quantizer_k_factor_rf" -> params.quantizerKFactorRF.toString, "quantizer_nprobe" -> params.quantizerNprobe.toString, "ht" -> params.ht.toString, "numberOfNeighbors" -> numOfNeighbors.toString, ) ).asInstanceOf[QueryTimeConfiguration[T, P]] } case _ => throw new IllegalArgumentException(s"server type: $algo is not supported yet") } queryTimeConfig } private def buildQueryable[T, P <: RuntimeParams, D <: Distance[D]]( inMemoryBuildRecorder: InMemoryLoadTestBuildRecorder ): Future[Queryable[T, P, D]] = { val queryable = loadTestType() match { case "remote" => { assert(serviceDestination().nonEmpty, "Service destination not defined") logger.info(s"Running load test with remote service ${serviceDestination()}") LoadTestUtils.buildRemoteServiceQueryClient[T, P, D]( serviceDestination(), "ann-load-test", statsReceiver, injector.instance[ServiceIdentifier], getRuntimeParamInjection[P], getDistanceInjection[D], getIndexIdInjection[T] ) } case "local" => { logger.info("Running load test locally..") assert(indexSetDir().nonEmpty, "Index set path is empty") val statsLoadTestBuildRecorder = new StatsLoadTestBuildRecorder(statsReceiver) val buildRecorder = new ComposedLoadTestBuildRecorder(Seq(inMemoryBuildRecorder, statsLoadTestBuildRecorder)) indexEmbeddingsAndGetQueryable[T, P, D]( buildRecorder, LoadTestUtils.getEmbeddingsSet(indexSetDir(), indexIdType()) ) } } queryable } private def indexEmbeddingsAndGetQueryable[T, P <: RuntimeParams, D <: Distance[D]]( buildRecorder: LoadTestBuildRecorder, indexSet: Seq[EntityEmbedding[T]] ): Future[Queryable[T, P, D]] = { logger.info(s"Indexing entity embeddings in index set with size ${indexSet.size}") val metric = getDistanceMetric[D] val indexIdInjection = getIndexIdInjection[T] val indexBuilder = new AnnIndexBuildLoadTest(buildRecorder) val appendable = algo() match { case "annoy" => assert(numOfTrees() > 0, "Must specify the number of trees for annoy") logger.info( s"Creating annoy index locally with num_of_trees: ${numOfTrees()}" ) TypedAnnoyIndex .indexBuilder( dimension(), numOfTrees(), metric, indexIdInjection, FuturePool.interruptibleUnboundedPool ) case "hnsw" => assert(efConstruction() > 0 && maxM() > 0, "Must specify ef_construction and max_m") logger.info( s"Creating hnsw index locally with max_m: ${maxM()} and ef_construction: ${efConstruction()}" ) TypedHnswIndex .index[T, D]( dimension(), metric, efConstruction(), maxM(), indexSet.size, ReadWriteFuturePool(FuturePool.interruptibleUnboundedPool) ) } indexBuilder .indexEmbeddings(appendable, indexSet, concurrencyLevel()) .asInstanceOf[Future[Queryable[T, P, D]]] } private[this] def performQueries[T, P <: RuntimeParams, D <: Distance[D]]( queryable: Queryable[T, P, D], queryTimeConfig: Seq[QueryTimeConfiguration[T, P]], queries: Seq[Query[T]] ): Future[Unit] = { val indexQuery = new AnnIndexQueryLoadTest() val duration = Duration(durationSec().toLong, TimeUnit.SECONDS) indexQuery.performQueries( queryable, targetQPS(), duration, queries, concurrencyLevel(), queryTimeConfig ) } // provide index id injection based on argument private[this] def getIndexIdInjection[T]: Injection[T, Array[Byte]] = { val injection = indexIdType() match { case "long" => AnnInjections.LongInjection case "string" => AnnInjections.StringInjection case "int" => AnnInjections.IntInjection case entityKind => EntityKind.getEntityKind(entityKind).byteInjection } injection.asInstanceOf[Injection[T, Array[Byte]]] } private[this] def getRuntimeParamInjection[ P <: RuntimeParams ]: Injection[P, ServiceRuntimeParams] = { val injection = algo() match { case "annoy" => AnnoyCommon.RuntimeParamsInjection case "hnsw" => HnswCommon.RuntimeParamsInjection case "faiss" => FaissCommon.RuntimeParamsInjection } injection.asInstanceOf[Injection[P, ServiceRuntimeParams]] } // provide distance injection based on argument private[this] def getDistanceInjection[D <: Distance[D]]: Injection[D, ServiceDistance] = { Metric.fromString(metric()).asInstanceOf[Injection[D, ServiceDistance]] } private[this] def getDistanceMetric[D <: Distance[D]]: Metric[D] = { Metric.fromString(metric()).asInstanceOf[Metric[D]] } private[this] def buildQueryTimeConfig[T, P <: RuntimeParams]( numOfNeighbors: Int, params: P, config: Map[String, String] ): QueryTimeConfiguration[T, P] = { val printableQueryRecorder = new InMemoryLoadTestQueryRecorder[T]() val scope = config.flatMap { case (key, value) => Seq(key, value.toString) }.toSeq val statsLoadTestQueryRecorder = new StatsLoadTestQueryRecorder[T]( // Put the run time params in the stats receiver names so that we can tell the difference when // we look at them later. statsReceiver.scope(algo()).scope(scope: _*) ) val queryRecorder = new ComposedLoadTestQueryRecorder( Seq(printableQueryRecorder, statsLoadTestQueryRecorder) ) QueryTimeConfiguration( queryRecorder, params, numOfNeighbors, printableQueryRecorder ) } override protected def modules: Seq[com.google.inject.Module] = Seq(ServiceIdentifierModule) } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/loadtest/AnnLoadTestWorker.scala ================================================ package com.twitter.ann.service.loadtest import com.google.common.annotations.VisibleForTesting import com.twitter.ann.common.Distance import com.twitter.ann.common.Queryable import com.twitter.ann.common.RuntimeParams import com.twitter.concurrent.AsyncMeter import com.twitter.concurrent.AsyncStream import com.twitter.finagle.util.DefaultTimer import com.twitter.logging.Logger import com.twitter.util.Duration import com.twitter.util.Future import com.twitter.util.Stopwatch import com.twitter.util.Timer import com.twitter.util.Try import java.util.concurrent.atomic.AtomicInteger object QueryTimeConfiguration { val ResultHeader = "params\tnumNeighbors\trecall@1\trecall@10\trecall\tavgLatencyMicros\tp50LatencyMicros\tp90LatencyMicros\tp99LatencyMicros\tavgRPS" } case class QueryTimeConfiguration[T, P <: RuntimeParams]( recorder: LoadTestQueryRecorder[T], param: P, numberOfNeighbors: Int, private val results: InMemoryLoadTestQueryRecorder[T]) { override def toString: String = s"QueryTimeConfiguration(param = $param, numberOfNeighbors = $numberOfNeighbors)" def printResults: String = { val snapshot = results.computeSnapshot() s"$param\t$numberOfNeighbors\t${results.top1Recall}\t${results.top10Recall}\t${results.recall}\t${snapshot.avgQueryLatencyMicros}\t${snapshot.p50QueryLatencyMicros}\t${snapshot.p90QueryLatencyMicros}\t${snapshot.p99QueryLatencyMicros}\t${results.avgRPS}" } } /** * Basic worker for ANN benchmark, send query with configured QPS and record results */ class AnnLoadTestWorker( timer: Timer = DefaultTimer) { private val logger = Logger() /** * @param queries List of tuple of query embedding and corresponding list of truth set of ids associated with the embedding * @param qps the maximum number of request per second to send to the queryable. Note that if you * do not set the concurrency level high enough you may not be able to achieve this. * @param duration how long to perform the load test. * @param configuration Query configuration encapsulating runtime params and recorder. * @param concurrencyLevel The maximum number of concurrent requests to the queryable at a time. * Note that you may not be able to achieve this number of concurrent * requests if you do not have the QPS set high enough. * * @return a Future that completes when the load test is over. It contains the number of requests * sent. */ def runWithQps[T, P <: RuntimeParams, D <: Distance[D]]( queryable: Queryable[T, P, D], queries: Seq[Query[T]], qps: Int, duration: Duration, configuration: QueryTimeConfiguration[T, P], concurrencyLevel: Int ): Future[Int] = { val elapsed = Stopwatch.start() val atomicInteger = new AtomicInteger(0) val fullStream = Stream.continually { if (elapsed() <= duration) { logger.ifDebug(s"running with config: $configuration") Some(atomicInteger.getAndIncrement() % queries.size) } else { logger.ifDebug(s"stopping with config: $configuration") None } } val limitedStream = fullStream.takeWhile(_.isDefined).flatten // at most we will have concurrencyLevel concurrent requests. So we should never have more than // concurrency level waiters. val asyncMeter = AsyncMeter.perSecond(qps, concurrencyLevel)(timer) Future.Unit.before { AsyncStream .fromSeq(limitedStream).mapConcurrent(concurrencyLevel) { index => asyncMeter.await(1).flatMap { _ => performQuery(configuration, queryable, queries(index)) } }.size } } @VisibleForTesting private[loadtest] def performQuery[T, P <: RuntimeParams, D <: Distance[D]]( configuration: QueryTimeConfiguration[T, P], queryable: Queryable[T, P, D], query: Query[T] ): Future[Try[Unit]] = { val elapsed = Stopwatch.start() queryable .query(query.embedding, configuration.numberOfNeighbors, configuration.param) .onSuccess { res: List[T] => // underneath LoadTestRecorder will record results for load test // knnMap should be truncated to be same size as query result configuration.recorder.recordQueryResult( query.trueNeighbours, res, elapsed.apply() ) logger.ifDebug(s"Successful query for $query") } .onFailure { e => logger.error(s"Failed query for $query: " + e) } .unit .liftToTry } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/loadtest/BUILD ================================================ scala_library( name = "loadtest", sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], tags = ["bazel-compatible"], dependencies = [ "ann/src/main/resources", "ann/src/main/scala/com/twitter/ann/annoy", "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/scala/com/twitter/ann/faiss", "ann/src/main/scala/com/twitter/ann/hnsw", "ann/src/main/scala/com/twitter/ann/util", "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", "finatra/inject/inject-server/src/main/scala", "src/scala/com/twitter/cortex/ml/embeddings/common:Helpers", "twitter-server-internal/src/main/scala", "util/util-logging/src/main/scala", ], ) jvm_binary( name = "bin", basename = "ann-loadtest", main = "com.twitter.ann.service.loadtest.AnnLoadTestMain", runtime_platform = "java11", dependencies = [ ":loadtest", "3rdparty/jvm/org/slf4j:slf4j-jdk14", "twitter-server/slf4j-jdk14/src/main/scala", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/loadtest/EmbeddingIndexer.scala ================================================ package com.twitter.ann.service.loadtest import com.twitter.ann.common.{Appendable, Distance, EntityEmbedding, Queryable, RuntimeParams} import com.twitter.ann.util.IndexBuilderUtils import com.twitter.util.{Future, Stopwatch} class EmbeddingIndexer { // Index embeddings into Appendable and return the (appendable, latency) pair // we need to return appendable itself here because for Annoy, we need to build // appendable and serialize it first, and then we could query with index directory // once we are confident to remove Annoy, should clean up this method. def indexEmbeddings[T, P <: RuntimeParams, D <: Distance[D]]( appendable: Appendable[T, P, D], recorder: LoadTestBuildRecorder, indexSet: Seq[EntityEmbedding[T]], concurrencyLevel: Int ): Future[Queryable[T, P, D]] = { val indexBuildingTimeElapsed = Stopwatch.start() val future = IndexBuilderUtils.addToIndex(appendable, indexSet, concurrencyLevel) future.map { _ => val indexBuildingTime = indexBuildingTimeElapsed() val toQueryableElapsed = Stopwatch.start() val queryable = appendable.toQueryable recorder.recordIndexCreation(indexSet.size, indexBuildingTime, toQueryableElapsed()) queryable } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/loadtest/LoadTestRecorder.scala ================================================ package com.twitter.ann.service.loadtest import com.google.common.util.concurrent.AtomicDouble import com.twitter.finagle.stats.{MetricsBucketedHistogram, Snapshot, StatsReceiver} import com.twitter.util.{Duration, Stopwatch} import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} trait LoadTestQueryRecorder[T] { def recordQueryResult( trueNeighbors: Seq[T], foundNeighbors: Seq[T], queryLatency: Duration ): Unit } case class LoadTestQueryResults( numResults: Int, top1Recall: Float, top10Recall: Option[Float], overallRecall: Float) private object LoadTestQueryRecorder { def recordQueryResult[T]( trueNeighbors: Seq[T], foundNeighbors: Seq[T] ): LoadTestQueryResults = { // record number of results returned val numResults = foundNeighbors.size if (trueNeighbors.isEmpty) { LoadTestQueryResults( numResults, 0f, Option.empty, 0f ) } else { // record top 1, top 10 and overall recall // recall here is computed as number of true neighbors within the returned points set // divides by the number of required neighbors val top1Recall = foundNeighbors.intersect(Seq(trueNeighbors.head)).size val top10Recall = if (numResults >= 10 && trueNeighbors.size >= 10) { Some( trueNeighbors.take(10).intersect(foundNeighbors).size.toFloat / 10 ) } else { None } val overallRecall = trueNeighbors .take(foundNeighbors.size).intersect(foundNeighbors).size.toFloat / Math.min(foundNeighbors.size, trueNeighbors.size) LoadTestQueryResults( numResults, top1Recall, top10Recall, overallRecall ) } } } class StatsLoadTestQueryRecorder[T]( statsReceiver: StatsReceiver) extends LoadTestQueryRecorder[T] { private[this] val numResultsStats = statsReceiver.stat("number_of_results") private[this] val recallStats = statsReceiver.stat("recall") private[this] val top1RecallStats = statsReceiver.stat("top_1_recall") private[this] val top10RecallStats = statsReceiver.stat("top_10_recall") private[this] val queryLatencyMicrosStats = statsReceiver.stat("query_latency_micros") override def recordQueryResult( trueNeighbors: Seq[T], foundNeighbors: Seq[T], queryLatency: Duration ): Unit = { val results = LoadTestQueryRecorder.recordQueryResult(trueNeighbors, foundNeighbors) numResultsStats.add(results.numResults) recallStats.add(results.overallRecall * 100) results.top10Recall.foreach { top10Recall => top10RecallStats.add(top10Recall * 100) } top1RecallStats.add(results.top1Recall * 100) queryLatencyMicrosStats.add(queryLatency.inMicroseconds) } } trait LoadTestBuildRecorder { def recordIndexCreation( indexSize: Int, indexLatency: Duration, toQueryableLatency: Duration ): Unit } class StatsLoadTestBuildRecorder( statsReceiver: StatsReceiver) extends LoadTestBuildRecorder { private[this] val indexLatencyGauge = statsReceiver.addGauge("index_latency_ms")(_) private[this] val indexSizeGauge = statsReceiver.addGauge("index_size")(_) private[this] val toQueryableGauge = statsReceiver.addGauge("to_queryable_latency_ms")(_) override def recordIndexCreation( indexSize: Int, indexLatency: Duration, toQueryableLatency: Duration ): Unit = { indexLatencyGauge(indexLatency.inMillis) indexSizeGauge(indexSize) toQueryableGauge(toQueryableLatency.inMillis) } } class QueryRecorderSnapshot(snapshot: Snapshot) { def avgQueryLatencyMicros: Double = snapshot.average def p50QueryLatencyMicros: Double = snapshot.percentiles.find(_.quantile == .5).get.value def p90QueryLatencyMicros: Double = snapshot.percentiles.find(_.quantile == .9).get.value def p99QueryLatencyMicros: Double = snapshot.percentiles.find(_.quantile == .99).get.value } class InMemoryLoadTestQueryRecorder[T]( // You have to specify a name of the histogram even though it is not used // Use latch period of bottom. We will compute a new snapshot every time we call computeSnapshot private[this] val latencyHistogram: MetricsBucketedHistogram = new MetricsBucketedHistogram("latencyhistogram", latchPeriod = Duration.Bottom)) extends LoadTestQueryRecorder[T] { private[this] val counter = new AtomicInteger(0) private[this] val countMoreThan10Results = new AtomicInteger(0) private[this] val recallSum = new AtomicDouble(0.0) private[this] val top1RecallSum = new AtomicDouble(0.0) private[this] val top10RecallSum = new AtomicDouble(0.0) private[this] val elapsedTimeFun = new AtomicReference[(Stopwatch.Elapsed, Duration)]() private[this] val elapsedTime = new AtomicReference[Duration](Duration.Zero) /** * Compute a snapshot of what happened between the time that this was called and the previous time * it was called. * @return */ def computeSnapshot(): QueryRecorderSnapshot = { new QueryRecorderSnapshot(latencyHistogram.snapshot()) } def recall: Double = if (counter.get() != 0) { recallSum.get * 100 / counter.get() } else { 0 } def top1Recall: Double = if (counter.get() != 0) { top1RecallSum.get * 100 / counter.get() } else { 0 } def top10Recall: Double = if (countMoreThan10Results.get() != 0) { top10RecallSum.get * 100 / countMoreThan10Results.get() } else { 0 } def avgRPS: Double = if (elapsedTime.get() != Duration.Zero) { (counter.get().toDouble * 1e9) / elapsedTime.get().inNanoseconds } else { 0 } override def recordQueryResult( trueNeighbors: Seq[T], foundNeighbors: Seq[T], queryLatency: Duration ): Unit = { elapsedTimeFun.compareAndSet(null, (Stopwatch.start(), queryLatency)) val results = LoadTestQueryRecorder.recordQueryResult(trueNeighbors, foundNeighbors) top1RecallSum.addAndGet(results.top1Recall) results.top10Recall.foreach { top10Recall => top10RecallSum.addAndGet(top10Recall) countMoreThan10Results.incrementAndGet() } recallSum.addAndGet(results.overallRecall) latencyHistogram.add(queryLatency.inMicroseconds) counter.incrementAndGet() // Requests are assumed to have started around the time time of the first time record was called // plus the time it took for that query to hhave completed. val (elapsedSinceFirstCall, firstQueryLatency) = elapsedTimeFun.get() val durationSoFar = elapsedSinceFirstCall() + firstQueryLatency elapsedTime.set(durationSoFar) } } class InMemoryLoadTestBuildRecorder extends LoadTestBuildRecorder { var indexLatency: Duration = Duration.Zero var indexSize: Int = 0 var toQueryableLatency: Duration = Duration.Zero override def recordIndexCreation( size: Int, indexLatencyArg: Duration, toQueryableLatencyArg: Duration ): Unit = { indexLatency = indexLatencyArg indexSize = size toQueryableLatency = toQueryableLatencyArg } } /** * A LoadTestRecorder that be composed by other recorders */ class ComposedLoadTestQueryRecorder[T]( recorders: Seq[LoadTestQueryRecorder[T]]) extends LoadTestQueryRecorder[T] { override def recordQueryResult( trueNeighbors: Seq[T], foundNeighbors: Seq[T], queryLatency: Duration ): Unit = recorders.foreach { _.recordQueryResult(trueNeighbors, foundNeighbors, queryLatency) } } /** * A LoadTestRecorder that be composed by other recorders */ class ComposedLoadTestBuildRecorder( recorders: Seq[LoadTestBuildRecorder]) extends LoadTestBuildRecorder { override def recordIndexCreation( indexSize: Int, indexLatency: Duration, toQueryableLatency: Duration ): Unit = recorders.foreach { _.recordIndexCreation(indexSize, indexLatency, toQueryableLatency) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/loadtest/LoadTestUtils.scala ================================================ package com.twitter.ann.service.loadtest import com.google.common.annotations.VisibleForTesting import com.twitter.ann.common.EmbeddingType.EmbeddingVector import com.twitter.ann.common.thriftscala.AnnQueryService import com.twitter.ann.common.thriftscala.NearestNeighborQuery import com.twitter.ann.common.thriftscala.NearestNeighborResult import com.twitter.ann.common.thriftscala.{Distance => ServiceDistance} import com.twitter.ann.common.thriftscala.{RuntimeParams => ServiceRuntimeParams} import com.twitter.ann.common.Distance import com.twitter.ann.common.EntityEmbedding import com.twitter.ann.common.Queryable import com.twitter.ann.common.RuntimeParams import com.twitter.ann.common.ServiceClientQueryable import com.twitter.bijection.Injection import com.twitter.cortex.ml.embeddings.common.EntityKind import com.twitter.finagle.builder.ClientBuilder import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.thrift.ClientId import com.twitter.finagle.Service import com.twitter.finagle.ThriftMux import com.twitter.ml.api.embedding.Embedding import com.twitter.search.common.file.AbstractFile.Filter import com.twitter.search.common.file.AbstractFile import com.twitter.search.common.file.FileUtils import com.twitter.search.common.file.LocalFile import com.twitter.util.Future import com.twitter.util.logging.Logger import java.io.File import scala.collection.JavaConversions._ import scala.collection.mutable import scala.util.Random object LoadTestUtils { lazy val Log = Logger(getClass.getName) private[this] val LocalPath = "." private[this] val RNG = new Random(100) private[loadtest] def getTruthSetMap[Q, I]( directory: String, queryIdType: String, indexIdType: String ): Map[Q, Seq[I]] = { Log.info(s"Loading truth set from ${directory}") val queryConverter = getKeyConverter[Q](queryIdType) val indexConverter = getKeyConverter[I](indexIdType) val res = loadKnnDirFileToMap( getLocalFileHandle(directory), // Knn truth file tsv format: [id neighbor:distance neighbor:distance ...] arr => { arr.map(str => indexConverter(str.substring(0, str.lastIndexOf(":")))).toSeq }, queryConverter ) assert(res.nonEmpty, s"Must have some something in the truth set ${directory}") res } private[this] def getLocalFileHandle( directory: String ): AbstractFile = { val fileHandle = FileUtils.getFileHandle(directory) if (fileHandle.isInstanceOf[LocalFile]) { fileHandle } else { val localFileHandle = FileUtils.getFileHandle(s"${LocalPath}${File.separator}${fileHandle.getName}") fileHandle.copyTo(localFileHandle) localFileHandle } } private[loadtest] def getEmbeddingsSet[T]( directory: String, idType: String ): Seq[EntityEmbedding[T]] = { Log.info(s"Loading embeddings from ${directory}") val res = loadKnnDirFileToMap( getLocalFileHandle(directory), arr => { arr.map(_.toFloat) }, getKeyConverter[T](idType) ).map { case (key, value) => EntityEmbedding[T](key, Embedding(value.toArray)) }.toSeq assert(res.nonEmpty, s"Must have some something in the embeddings set ${directory}") res } private[this] def loadKnnDirFileToMap[K, V]( directory: AbstractFile, f: Array[String] => Seq[V], converter: String => K ): Map[K, Seq[V]] = { val map = mutable.HashMap[K, Seq[V]]() directory .listFiles(new Filter { override def accept(file: AbstractFile): Boolean = file.getName != AbstractFile.SUCCESS_FILE_NAME }).foreach { file => asScalaBuffer(file.readLines()).foreach { line => addToMapFromKnnString(line, f, map, converter) } } map.toMap } // Generating random float with value range bounded between minValue and maxValue private[loadtest] def getRandomQuerySet( dimension: Int, totalQueries: Int, minValue: Float, maxValue: Float ): Seq[EmbeddingVector] = { Log.info( s"Generating $totalQueries random queries for dimension $dimension with value between $minValue and $maxValue...") assert(totalQueries > 0, s"Total random queries $totalQueries should be greater than 0") assert( maxValue > minValue, s"Random embedding max value should be greater than min value. min: $minValue max: $maxValue") (1 to totalQueries).map { _ => val embedding = Array.fill(dimension)(minValue + (maxValue - minValue) * RNG.nextFloat()) Embedding(embedding) } } private[this] def getKeyConverter[T](idType: String): String => T = { val converter = idType match { case "long" => (s: String) => s.toLong case "string" => (s: String) => s case "int" => (s: String) => s.toInt case entityKind => (s: String) => EntityKind.getEntityKind(entityKind).stringInjection.invert(s).get } converter.asInstanceOf[String => T] } private[loadtest] def buildRemoteServiceQueryClient[T, P <: RuntimeParams, D <: Distance[D]]( destination: String, clientId: String, statsReceiver: StatsReceiver, serviceIdentifier: ServiceIdentifier, runtimeParamInjection: Injection[P, ServiceRuntimeParams], distanceInjection: Injection[D, ServiceDistance], indexIdInjection: Injection[T, Array[Byte]] ): Future[Queryable[T, P, D]] = { val client: AnnQueryService.MethodPerEndpoint = new AnnQueryService.FinagledClient( service = ClientBuilder() .reportTo(statsReceiver) .dest(destination) .stack(ThriftMux.client.withMutualTls(serviceIdentifier).withClientId(ClientId(clientId))) .build(), stats = statsReceiver ) val service = new Service[NearestNeighborQuery, NearestNeighborResult] { override def apply(request: NearestNeighborQuery): Future[NearestNeighborResult] = client.query(request) } Future.value( new ServiceClientQueryable[T, P, D]( service, runtimeParamInjection, distanceInjection, indexIdInjection ) ) } // helper method to convert a line in KNN file output format into map @VisibleForTesting def addToMapFromKnnString[K, V]( line: String, f: Array[String] => Seq[V], map: mutable.HashMap[K, Seq[V]], converter: String => K ): Unit = { val items = line.split("\t") map += converter(items(0)) -> f(items.drop(1)) } def printResults( inMemoryBuildRecorder: InMemoryLoadTestBuildRecorder, queryTimeConfigurations: Seq[QueryTimeConfiguration[_, _]] ): Seq[String] = { val queryTimeConfigStrings = queryTimeConfigurations.map { config => config.printResults } Seq( "Build results", "indexingTimeSecs\ttoQueryableTimeMs\tindexSize", s"${inMemoryBuildRecorder.indexLatency.inSeconds}\t${inMemoryBuildRecorder.toQueryableLatency.inMilliseconds}\t${inMemoryBuildRecorder.indexSize}", "Query results", QueryTimeConfiguration.ResultHeader ) ++ queryTimeConfigStrings } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/loadtest/README.md ================================================ # Loadtest ANN query service with random embeddings An ANN query service can be load-tested with random embeddings as queries, generated automatically by loadtest tool. Example script to load test a ANN query service with random embeddings: ```bash $ aurora job create smf1//staging/ann-loadtest-service ann/src/main/aurora/loadtest/loadtest.aurora \ --bind=profile.name=ann-loadtest-service \ --bind=profile.role= \ --bind=profile.duration_sec=10 \ --bind=profile.number_of_neighbors=10 \ --bind=profile.qps=200 \ --bind=profile.algo=hnsw \ --bind=profile.metric=Cosine \ --bind=profile.index_id_type=int \ --bind=profile.hnsw_ef=400,600,800 \ --bind=profile.embedding_dimension=3 \ --bind=profile.concurrency_level=8 \ --bind=profile.loadtest_type=remote \ --bind=profile.service_destination=/srv#/staging/local/apoorvs/ann-server-test \ --bind=profile.with_random_queries=True \ --bind=profile.random_queries_count=50000 \ --bind=profile.random_embedding_min_value=-10.0 \ --bind=profile.random_embedding_max_value=10.0 ``` It will run the loadtest with `50000` random embeddings, where each embedding value will be range bounded between `random_embedding_min_value` and `random_embedding_max_value`. In the above the case it will be bounded between `-10.0` and `10.0`. If `random_embedding_min_value` and `random_embedding_max_value` are not supplied default value of `-1.0` and `1.0` will be used. ## Results Load test results will be printed to stdout of an aurora job. # Loadtest ANN query service with query set An ANN query service can be load-tested with sample queries drawn from the embeddings dataset. For creating sample queries i.e `query_set` refer this [section](#query-set-generator). Test is run with `live` version of loadtest binary that is already available in packer. Example script to load test a ANN query service: ```bash $ aurora job create smf1//staging/ann-loadtest-service ann/src/main/aurora/loadtest/loadtest.aurora \ --bind=profile.name=ann-loadtest-service \ --bind=profile.role= \ --bind=profile.duration_sec=10 \ --bind=profile.query_set_dir=hdfs:///user/cortex/ann_example/dataset/search/query_knn/query_set \ --bind=profile.number_of_neighbors=10 \ --bind=profile.qps=200 \ --bind=profile.algo=hnsw \ --bind=profile.query_id_type=string \ --bind=profile.index_id_type=string \ --bind=profile.metric=Cosine \ --bind=profile.hnsw_ef=400,600,800 \ --bind=profile.embedding_dimension=100 \ --bind=profile.concurrency_level=8 \ --bind=profile.loadtest_type=remote \ --bind=profile.service_destination=/srv#/staging/local/apoorvs/ann-server-test ``` # In-Memory based loadtest for measuring recall Load test can be with the above created dataset in memory. For running in in-memory mode, index is created in memory, and for that you need `query_set/index_set/truth_set`. For creating this dataset refer this [section](#knn-truth-set-generator). Test is run with `live` version loadtest binary that is already available in packer. Example script In-Memory index building and benchmarking: ```bash $ aurora job create smf1//staging/ann-loadtest ann/src/main/aurora/loadtest/loadtest.aurora \ --bind=profile.name=ann-loadtest \ --bind=profile.role= \ --bind=profile.duration_sec=10 \ --bind=profile.truth_set_dir=hdfs:///user/cortex/ann_example/dataset/search/query_knn/true_knn \ --bind=profile.query_set_dir=hdfs:///user/cortex/ann_example/dataset/search/query_knn/query_set \ --bind=profile.index_set_dir=hdfs:///user/cortex/ann_example/dataset/search/query_knn/index_set \ --bind=profile.number_of_neighbors=10 \ --bind=profile.qps=200 \ --bind=profile.algo=hnsw \ --bind=profile.query_id_type=string \ --bind=profile.index_id_type=string \ --bind=profile.metric=Cosine \ --bind=profile.hnsw_ef_construction=15 \ --bind=profile.hnsw_max_m=10 \ --bind=profile.hnsw_ef=400,600,800 \ --bind=profile.embedding_dimension=100 \ --bind=profile.concurrency_level=8 \ --bind=profile.loadtest_type=local ``` # Loadtest faiss ```bash $ aurora job create smf1//staging/ann-loadtest-service ann/src/main/aurora/loadtest/loadtest.aurora \ --bind=profile.name=ann-loadtest-service \ --bind=profile.role= \ --bind=profile.duration_sec=10 \ --bind=profile.number_of_neighbors=10 \ --bind=profile.qps=200 \ --bind=profile.algo=faiss \ # Changed to faiss --bind=profile.faiss_nprobe=1,3,9,27,81,128,256,512 \ # Added --bind=profile.faiss_quantizerKfactorRF=1,2 \ # Pass a list to do grid search --bind=profile.faiss_quantizerNprobe=128 \ # Added --bind=profile.metric=Cosine \ --bind=profile.index_id_type=int \ --bind=profile.embedding_dimension=3 \ --bind=profile.concurrency_level=8 \ --bind=profile.loadtest_type=remote \ --bind=profile.service_destination=/srv#/staging/local/apoorvs/ann-server-test \ --bind=profile.with_random_queries=True \ --bind=profile.random_queries_count=50000 \ --bind=profile.random_embedding_min_value=-10.0 \ --bind=profile.random_embedding_max_value=10.0 ``` Full list of faiss specific parameters. [Exact definition of all available parameters](https://github.com/facebookresearch/faiss/blob/36f2998a6469280cef3b0afcde2036935a29aa1f/faiss/AutoTune.cpp#L444). Please reach out if you need to use parameters which aren't shown below ``` faiss_nprobe = Default(String, '1') faiss_quantizerEf = Default(String, '0') faiss_quantizerKfactorRF = Default(String, '0') faiss_quantizerNprobe = Default(String, '0') faiss_ht = Default(String, '0') ``` # Query Set Generator Sample queries can be generated from the embeddings dataset and can be used directly with load test in tab format. To generate sample queries `EmbeddingSamplingJob` can be used as follows. ```bash $ ./bazel bundle cortex-core/entity-embeddings/src/scala/main/com/twitter/scalding/util/EmbeddingFormat:embeddingformat-deploy $ export INPUT_PATH=/user/cortex/embeddings/user/tfwproducersg/embedding_datarecords_on_data/2018/05/01 $ export ENTITY_KIND=user $ export EMBEDDING_INPUT_FORMAT=usertensor $ export OUTPUT_PATH=/user/$USER/sample_embeddings $ export SAMPLE_PERCENT=0.1 $ oscar hdfs \ --screen --tee log.txt \ --hadoop-client-memory 6000 \ --hadoop-properties "yarn.app.mapreduce.am.resource.mb=6000;yarn.app.mapreduce.am.command-opts='-Xmx7500m';mapreduce.map.memory.mb=7500;mapreduce.reduce.java.opts='-Xmx6000m';mapreduce.reduce.memory.mb=7500;mapred.task.timeout=36000000;" \ --min-split-size 284217728 \ --bundle embeddingformat-deploy \ --host hadoopnest1.smf1.twitter.com \ --tool com.twitter.scalding.entityembeddings.util.EmbeddingFormat.EmbeddingSamplingJob -- \ --entity_kind $ENTITY_KIND \ --input.embedding_path $INPUT_PATH \ --input.embedding_format $EMBEDDING_INPUT_FORMAT \ --output.embedding_path $OUTPUT_PATH \ --output.embedding_format tab \ --sample_percent $SAMPLE_PERCENT ``` It will sample 0.1% of embeddings and store them in `tab` format to hdfs that can be direcly used as `query_set` for loadtest. # Knn Truth Set Generator To use load test framework to benchmark recall, you need to split your data set into index_set, query_set and knn_truth - index_set: data that will be indexed for ann - query_set: data that will be used for queries - truth_set: the real nearest neighbor used as truth to compute recall And also you need to figure out the dimension for your embedding vectors. KnnTruthSetGenerator can help to prepare data sets: ```bash $ ./bazel bundle ann/src/main/scala/com/twitter/ann/scalding/offline:ann-offline-deploy $ export QUERY_EMBEDDINGS_PATH=/user/cortex-mlx/official_examples/ann/non_pii_random_user_embeddings_tab_format $ export INDEX_EMBEDDINGS_PATH=/user/cortex-mlx/official_examples/ann/non_pii_random_user_embeddings_tab_format $ export TRUTH_SET_PATH=/user/$USER/truth_set $ export INDEX_SET_PATH=/user/$USER/index_set $ export QUERY_SET_PATH=/user/$USER/query_set $ export METRIC=InnerProduct $ export QUERY_ENTITY_KIND=user $ export INDEX_ENTITY_KIND=user $ export NEIGHBOURS=10 $ oscar hdfs \ --screen --tee log.txt \ --hadoop-client-memory 6000 \ --hadoop-properties "yarn.app.mapreduce.am.resource.mb=6000;yarn.app.mapreduce.am.command-opts='-Xmx7500m';mapreduce.map.memory.mb=7500;mapreduce.reduce.java.opts='-Xmx6000m';mapreduce.reduce.memory.mb=7500;mapred.task.timeout=36000000;" \ --bundle ann-offline-deploy \ --min-split-size 284217728 \ --host hadoopnest1.smf1.twitter.com \ --tool com.twitter.ann.scalding.offline.KnnTruthSetGenerator -- \ --neighbors $NEIGHBOURS \ --metric $METRIC \ --query_entity_kind $QUERY_ENTITY_KIND \ --query.embedding_path $QUERY_EMBEDDINGS_PATH \ --query.embedding_format tab \ --query_sample_percent 50.0 \ --index_entity_kind $INDEX_ENTITY_KIND \ --index.embedding_path $INDEX_EMBEDDINGS_PATH \ --index.embedding_format tab \ --index_sample_percent 90.0 \ --query_set_output.embedding_path $QUERY_SET_PATH \ --query_set_output.embedding_format tab \ --index_set_output.embedding_path $INDEX_SET_PATH \ --index_set_output.embedding_format tab \ --truth_set_output_path $TRUTH_SET_PATH \ --reducers 100 ``` It will sample 90% of index set embeddings and 50% of query embeddings from total and then it will generate 3 datasets from the same that are index set, query set and true nearest neighbours from query to index in the tab format. `Note`: The reason for using high sample percent is due to the fact the sample embeddings dataset is small. For real use cases query set should be really small. Set `--reducers` according to the embeddings dataset size. # FAQ There are multiple type of `query_id_type` and `index_id_type` that can be used. Some native types like string/int/long or related to entity embeddings like tweet/word/user/url... for more info: [Link](https://cgit.twitter.biz/source/tree/src/scala/com/twitter/cortex/ml/embeddings/common/EntityKind.scala#n8) ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/BUILD ================================================ scala_library( name = "common", sources = [ "BaseQueryIndexServer.scala", "Exceptions.scala", "QueryIndexThriftController.scala", "QueryableProvider.scala", "RefreshableQueryable.scala", "UnsafeQueryIndexServer.scala", ], compiler_option_sets = ["fatal_warnings"], tags = ["bazel-compatible"], dependencies = [ ":index_path_provider", "3rdparty/jvm/com/google/guava", "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/javax/inject:javax.inject", "3rdparty/jvm/net/codingwell:scala-guice", "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", "finagle/finagle-core/src/main", "finagle/finagle-zipkin-scribe", "finatra-internal/decider", "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-core/src/main/scala", "mediaservices/commons", "scrooge/scrooge-core/src/main/scala", "src/scala/com/twitter/cortex/ml/embeddings/common:Helpers", "util/util-app/src/main/scala", "util/util-logging/src/main/scala", ], ) scala_library( name = "index_path_provider", sources = [ "IndexPathProvider.scala", "QueryServerUtil.scala", ], compiler_option_sets = ["fatal_warnings"], tags = ["bazel-compatible"], dependencies = [ "ann/src/main/scala/com/twitter/ann/hnsw", "src/java/com/twitter/search/common/file", "util/util-logging/src/main/scala", "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], ) scala_library( name = "faiss_index_path_provider", sources = ["FaissIndexPathProvider.scala"], compiler_option_sets = ["fatal_warnings"], tags = ["bazel-compatible"], dependencies = [ ":index_path_provider", "ann/src/main/scala/com/twitter/ann/faiss", "src/java/com/twitter/search/common/file", "util/util-logging/src/main/scala", "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/BaseQueryIndexServer.scala ================================================ package com.twitter.ann.service.query_server.common import com.google.inject.Module import com.twitter.ann.common.thriftscala.AnnQueryService import com.twitter.app.Flag import com.twitter.finatra.decider.modules.DeciderModule import com.twitter.finatra.thrift.ThriftServer import com.twitter.finatra.mtls.thriftmux.Mtls import com.twitter.finatra.mtls.thriftmux.modules.MtlsThriftWebFormsModule import com.twitter.finatra.thrift.filters.{ AccessLoggingFilter, LoggingMDCFilter, StatsFilter, ThriftMDCFilter, TraceIdMDCFilter } import com.twitter.finatra.thrift.routing.ThriftRouter /** * This class provides most of the configuration needed for logging, stats, deciders etc. */ abstract class BaseQueryIndexServer extends ThriftServer with Mtls { protected val environment: Flag[String] = flag[String]("environment", "service environment") /** * Override with method to provide more module to guice. */ protected def additionalModules: Seq[Module] /** * Override this method to add the controller to the thrift router. BaseQueryIndexServer takes * care of most of the other configuration for you. * @param router */ protected def addController(router: ThriftRouter): Unit override protected final lazy val modules: Seq[Module] = Seq( DeciderModule, new MtlsThriftWebFormsModule[AnnQueryService.MethodPerEndpoint](this) ) ++ additionalModules override protected final def configureThrift(router: ThriftRouter): Unit = { router .filter[LoggingMDCFilter] .filter[TraceIdMDCFilter] .filter[ThriftMDCFilter] .filter[AccessLoggingFilter] .filter[StatsFilter] addController(router) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/Exceptions.scala ================================================ package com.twitter.ann.service.query_server.common import com.twitter.ann.common.thriftscala.BadRequest import com.twitter.mediaservices.commons._ object RuntimeExceptionTransform extends ExceptionTransformer { override def transform = { case e: BadRequest => MisuseExceptionInfo(e) } override def getStatName: PartialFunction[Exception, String] = { case e: BadRequest => exceptionName(e, e.code.name) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/FaissIndexPathProvider.scala ================================================ package com.twitter.ann.service.query_server.common import com.twitter.finagle.stats.StatsReceiver import com.twitter.logging.Logger import com.twitter.search.common.file.AbstractFile case class FaissIndexPathProvider( override val minIndexSizeBytes: Long, override val maxIndexSizeBytes: Long, override val statsReceiver: StatsReceiver) extends BaseIndexPathProvider { override val log = Logger.get("FAISSIndexPathProvider") override def isValidIndex(dir: AbstractFile): Boolean = { dir.isDirectory && dir.hasSuccessFile && dir.getChild("faiss.index").exists() } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/IndexPathProvider.scala ================================================ package com.twitter.ann.service.query_server.common import com.twitter.ann.common.IndexOutputFile import com.twitter.ann.hnsw.HnswCommon._ import com.twitter.finagle.stats.StatsReceiver import com.twitter.logging.Logger import com.twitter.search.common.file.AbstractFile import com.twitter.search.common.file.AbstractFile.Filter import com.twitter.search.common.file.PathUtils import com.twitter.util.Try import java.io.IOException import java.util.concurrent.atomic.AtomicReference import scala.collection.JavaConverters._ import scala.math.Ordering.comparatorToOrdering abstract class IndexPathProvider { def provideIndexPath(rootPath: AbstractFile, group: Boolean = false): Try[AbstractFile] def provideIndexPathWithGroups(rootPath: AbstractFile): Try[Seq[AbstractFile]] } abstract class BaseIndexPathProvider extends IndexPathProvider { protected val minIndexSizeBytes: Long protected val maxIndexSizeBytes: Long protected val statsReceiver: StatsReceiver protected val log: Logger private val invalidPathCounter = statsReceiver.counter("invalid_index") private val failToLocateDirectoryCounter = statsReceiver.counter("find_latest_path_fail") private val successProvidePathCounter = statsReceiver.counter("provide_path_success") private val latestGroupCount = new AtomicReference(0f) private val latestIndexTimestamp = new AtomicReference(0f) private val latestValidIndexTimestamp = new AtomicReference(0f) private val INDEX_METADATA_FILE = "ANN_INDEX_METADATA" private val latestIndexGauge = statsReceiver.addGauge("latest_index_timestamp")( latestIndexTimestamp.get() ) private val latestValidIndexGauge = statsReceiver.addGauge("latest_valid_index_timestamp")( latestValidIndexTimestamp.get() ) private val latestGroupCountGauge = statsReceiver.addGauge("latest_group_count")( latestGroupCount.get() ) private val latestTimeStampDirectoryFilter = new AbstractFile.Filter { /** Determines which files should be accepted when listing a directory. */ override def accept(file: AbstractFile): Boolean = { val name = file.getName PathUtils.TIMESTAMP_PATTERN.matcher(name).matches() } } private def findLatestTimeStampValidSuccessDirectory( path: AbstractFile, group: Boolean ): AbstractFile = { log.info(s"Calling findLatestTimeStampValidSuccessDirectory with ${path.getPath}") // Get all the timestamp directories val dateDirs = path.listFiles(latestTimeStampDirectoryFilter).asScala.toSeq if (dateDirs.nonEmpty) { // Validate the indexes val latestValidPath = { if (group) { // For grouped, check all the individual group indexes and stop as soon as a valid index // is found. dateDirs .sorted(comparatorToOrdering(PathUtils.NEWEST_FIRST_COMPARATOR)).find(file => { val indexMetadataFile = file.getChild(INDEX_METADATA_FILE) val indexes = file.listFiles().asScala.filter(_.isDirectory) val isValid = if (indexMetadataFile.exists()) { // Metadata file exists. Check the number of groups and verify the index is // complete val indexMetadata = new IndexOutputFile(indexMetadataFile).loadIndexMetadata() if (indexMetadata.numGroups.get != indexes.size) { log.info( s"Grouped index ${file.getPath} should have ${indexMetadata.numGroups.get} groups but had ${indexes.size}") } indexMetadata.numGroups.get == indexes.size } else { // True if the file doesn't exist. This is to make this change backwards // compatible for clients using the old version of the dataflow job true } isValid && indexes.forall(index => { index.hasSuccessFile && isValidIndex(index) && QueryServerUtil .isValidIndexDirSize(index, minIndexSizeBytes, maxIndexSizeBytes) }) }) } else { // For non-grouped, find the first valid index. dateDirs .sorted(comparatorToOrdering(PathUtils.NEWEST_FIRST_COMPARATOR)).find(file => { file.hasSuccessFile && QueryServerUtil .isValidIndexDirSize(file, minIndexSizeBytes, maxIndexSizeBytes) }) } } if (latestValidPath.nonEmpty) { // Log the results successProvidePathCounter.incr() if (group) { latestGroupCount.set(latestValidPath.get.listFiles().asScala.count(_.isDirectory)) log.info( s"findLatestTimeStampValidSuccessDirectory latestValidPath ${latestValidPath.get.getPath} and number of groups $latestGroupCount") } else { val latestValidPathSize = latestValidPath.get.listFiles(true).asScala.map(_.getSizeInBytes).sum log.info( s"findLatestTimeStampValidSuccessDirectory latestValidPath ${latestValidPath.get.getPath} and size $latestValidPathSize") } return latestValidPath.get } } // Fail if no index or no valid index. failToLocateDirectoryCounter.incr() throw new IOException(s"Cannot find any valid directory with SUCCESS file at ${path.getName}") } def isValidIndex(index: AbstractFile): Boolean override def provideIndexPath( rootPath: AbstractFile, group: Boolean = false ): Try[AbstractFile] = { Try { val latestValidPath = findLatestTimeStampValidSuccessDirectory(rootPath, group) if (!group) { val latestPath = PathUtils.findLatestTimeStampSuccessDirectory(rootPath) // since latestValidPath does not throw exception, latestPath must exist assert(latestPath.isPresent) val latestPathSize = latestPath.get.listFiles(true).asScala.map(_.getSizeInBytes).sum log.info(s"provideIndexPath latestPath ${latestPath .get() .getPath} and size $latestPathSize") latestIndexTimestamp.set(latestPath.get().getName.toFloat) // latest directory is not valid, update counter for alerts if (latestPath.get() != latestValidPath) { invalidPathCounter.incr() } } else { latestIndexTimestamp.set(latestValidPath.getName.toFloat) } latestValidIndexTimestamp.set(latestValidPath.getName.toFloat) latestValidPath } } override def provideIndexPathWithGroups( rootPath: AbstractFile ): Try[Seq[AbstractFile]] = { val latestValidPath = provideIndexPath(rootPath, true) latestValidPath.map { path => path .listFiles(new Filter { override def accept(file: AbstractFile): Boolean = file.isDirectory && file.hasSuccessFile }).asScala.toSeq } } } case class ValidatedIndexPathProvider( override val minIndexSizeBytes: Long, override val maxIndexSizeBytes: Long, override val statsReceiver: StatsReceiver) extends BaseIndexPathProvider { override val log = Logger.get("ValidatedIndexPathProvider") override def isValidIndex(dir: AbstractFile): Boolean = { isValidHnswIndex(dir) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/QueryIndexThriftController.scala ================================================ package com.twitter.ann.service.query_server.common import com.twitter.ann.common._ import com.twitter.ann.common.EmbeddingType._ import com.twitter.ann.common.thriftscala.AnnQueryService.Query import com.twitter.ann.common.thriftscala.AnnQueryService import com.twitter.ann.common.thriftscala.NearestNeighbor import com.twitter.ann.common.thriftscala.NearestNeighborResult import com.twitter.ann.common.thriftscala.{Distance => ServiceDistance} import com.twitter.ann.common.thriftscala.{RuntimeParams => ServiceRuntimeParams} import com.twitter.bijection.Injection import com.twitter.finagle.Service import com.twitter.finagle.stats.StatsReceiver import com.twitter.finatra.thrift.Controller import com.twitter.mediaservices.commons.{ThriftServer => TServer} import java.nio.ByteBuffer import javax.inject.Inject class QueryIndexThriftController[T, P <: RuntimeParams, D <: Distance[D]] @Inject() ( statsReceiver: StatsReceiver, queryable: Queryable[T, P, D], runtimeParamInjection: Injection[P, ServiceRuntimeParams], distanceInjection: Injection[D, ServiceDistance], idInjection: Injection[T, Array[Byte]]) extends Controller(AnnQueryService) { private[this] val thriftServer = new TServer(statsReceiver, Some(RuntimeExceptionTransform)) val trackingStatName = "ann_query" private[this] val stats = statsReceiver.scope(trackingStatName) private[this] val numOfNeighboursRequested = stats.stat("num_of_neighbours_requested") private[this] val numOfNeighboursResponse = stats.stat("num_of_neighbours_response") private[this] val queryKeyNotFound = stats.stat("query_key_not_found") /** * Implements AnnQueryService.query, returns nearest neighbours for a given query */ val query: Service[Query.Args, Query.SuccessType] = { args: Query.Args => thriftServer.track(trackingStatName) { val query = args.query val key = query.key val embedding = embeddingSerDe.fromThrift(query.embedding) val numOfNeighbours = query.numberOfNeighbors val withDistance = query.withDistance val runtimeParams = runtimeParamInjection.invert(query.runtimeParams).get numOfNeighboursRequested.add(numOfNeighbours) val result = if (withDistance) { val nearestNeighbors = if (queryable.isInstanceOf[QueryableGrouped[T, P, D]]) { queryable .asInstanceOf[QueryableGrouped[T, P, D]] .queryWithDistance(embedding, numOfNeighbours, runtimeParams, key) } else { queryable .queryWithDistance(embedding, numOfNeighbours, runtimeParams) } nearestNeighbors.map { list => list.map { nn => NearestNeighbor( ByteBuffer.wrap(idInjection.apply(nn.neighbor)), Some(distanceInjection.apply(nn.distance)) ) } } } else { val nearestNeighbors = if (queryable.isInstanceOf[QueryableGrouped[T, P, D]]) { queryable .asInstanceOf[QueryableGrouped[T, P, D]] .query(embedding, numOfNeighbours, runtimeParams, key) } else { queryable .query(embedding, numOfNeighbours, runtimeParams) } nearestNeighbors .map { list => list.map { nn => NearestNeighbor(ByteBuffer.wrap(idInjection.apply(nn))) } } } result.map(NearestNeighborResult(_)).onSuccess { r => numOfNeighboursResponse.add(r.nearestNeighbors.size) } } } handle(Query) { query } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/QueryServerUtil.scala ================================================ package com.twitter.ann.service.query_server.common import com.twitter.logging.Logger import com.twitter.search.common.file.AbstractFile import scala.collection.JavaConverters._ object QueryServerUtil { private val log = Logger.get("QueryServerUtil") /** * Validate if the abstract file (directory) size is within the defined limits. * @param dir Hdfs/Local directory * @param minIndexSizeBytes minimum size of file in bytes (Exclusive) * @param maxIndexSizeBytes minimum size of file in bytes (Exclusive) * @return true if file size within minIndexSizeBytes and maxIndexSizeBytes else false */ def isValidIndexDirSize( dir: AbstractFile, minIndexSizeBytes: Long, maxIndexSizeBytes: Long ): Boolean = { val recursive = true val dirSize = dir.listFiles(recursive).asScala.map(_.getSizeInBytes).sum log.debug(s"Ann index directory ${dir.getPath} size in bytes $dirSize") val isValid = (dirSize > minIndexSizeBytes) && (dirSize < maxIndexSizeBytes) if (!isValid) { log.info(s"Ann index directory is invalid ${dir.getPath} size in bytes $dirSize") } isValid } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/QueryableProvider.scala ================================================ package com.twitter.ann.service.query_server.common import com.twitter.ann.common.{Distance, Queryable, RuntimeParams} import com.twitter.search.common.file.AbstractFile trait QueryableProvider[T, P <: RuntimeParams, D <: Distance[D]] { def provideQueryable(indexDir: AbstractFile): Queryable[T, P, D] } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/RefreshableQueryable.scala ================================================ package com.twitter.ann.service.query_server.common import com.google.common.annotations.VisibleForTesting import com.google.common.util.concurrent.ThreadFactoryBuilder import com.twitter.ann.common.EmbeddingType.EmbeddingVector import com.twitter.ann.common.Distance import com.twitter.ann.common.NeighborWithDistance import com.twitter.ann.common.Queryable import com.twitter.ann.common.QueryableGrouped import com.twitter.ann.common.RuntimeParams import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.StatsReceiver import com.twitter.logging.Logger import com.twitter.search.common.file.AbstractFile import com.twitter.util.Duration import com.twitter.util.Future import java.util.concurrent.atomic.AtomicReference import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import scala.util.Random import scala.util.control.NonFatal class RefreshableQueryable[T, P <: RuntimeParams, D <: Distance[D]]( grouped: Boolean, rootDir: AbstractFile, queryableProvider: QueryableProvider[T, P, D], indexPathProvider: IndexPathProvider, statsReceiver: StatsReceiver, updateInterval: Duration = 10.minutes) extends QueryableGrouped[T, P, D] { private val log = Logger.get("RefreshableQueryable") private val loadCounter = statsReceiver.counter("load") private val loadFailCounter = statsReceiver.counter("load_error") private val newIndexCounter = statsReceiver.counter("new_index") protected val random = new Random(System.currentTimeMillis()) private val threadFactory = new ThreadFactoryBuilder() .setNameFormat("refreshable-queryable-update-%d") .build() // single thread to check and load index private val executor = Executors.newScheduledThreadPool(1, threadFactory) private[common] val indexPathRef: AtomicReference[AbstractFile] = new AtomicReference(indexPathProvider.provideIndexPath(rootDir, grouped).get()) private[common] val queryableRef: AtomicReference[Map[Option[String], Queryable[T, P, D]]] = { if (grouped) { val mapping = getGroupMapping new AtomicReference(mapping) } else { new AtomicReference(Map(None -> buildIndex(indexPathRef.get()))) } } private val servingIndexGauge = statsReceiver.addGauge("serving_index_timestamp") { indexPathRef.get().getName.toFloat } log.info("System.gc() before start") System.gc() private val reloadTask = new Runnable { override def run(): Unit = { innerLoad() } } def start(): Unit = { executor.scheduleWithFixedDelay( reloadTask, // init reloading with random delay computeRandomInitDelay().inSeconds, updateInterval.inSeconds, TimeUnit.SECONDS ) } private def buildIndex(indexPath: AbstractFile): Queryable[T, P, D] = { log.info(s"build index from ${indexPath.getPath}") queryableProvider.provideQueryable(indexPath) } @VisibleForTesting private[common] def innerLoad(): Unit = { log.info("Check and load for new index") loadCounter.incr() try { // Find the latest directory val latestPath = indexPathProvider.provideIndexPath(rootDir, grouped).get() if (indexPathRef.get() != latestPath) { log.info(s"loading index from: ${latestPath.getName}") newIndexCounter.incr() if (grouped) { val mapping = getGroupMapping queryableRef.set(mapping) } else { val queryable = buildIndex(latestPath) queryableRef.set(Map(None -> queryable)) } indexPathRef.set(latestPath) } else { log.info(s"Current index already up to date: ${indexPathRef.get.getName}") } } catch { case NonFatal(err) => loadFailCounter.incr() log.error(s"Failed to load index: $err") } log.info(s"Current index loaded from ${indexPathRef.get().getPath}") } @VisibleForTesting private[common] def computeRandomInitDelay(): Duration = { val bound = 5.minutes val nextUpdateSec = updateInterval + Duration.fromSeconds( random.nextInt(bound.inSeconds) ) nextUpdateSec } /** * ANN query for ids with key as group id * @param embedding: Embedding/Vector to be queried with. * @param numOfNeighbors: Number of neighbours to be queried for. * @param runtimeParams: Runtime params associated with index to control accuracy/latency etc. * @param key: Optional key to lookup specific ANN index and perform query there * @return List of approximate nearest neighbour ids. */ override def query( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P, key: Option[String] ): Future[List[T]] = { val mapping = queryableRef.get() if (!mapping.contains(key)) { Future.value(List()) } else { mapping.get(key).get.query(embedding, numOfNeighbors, runtimeParams) } } /** * ANN query for ids with key as group id with distance * @param embedding: Embedding/Vector to be queried with. * @param numOfNeighbors: Number of neighbours to be queried for. * @param runtimeParams: Runtime params associated with index to control accuracy/latency etc. * @param key: Optional key to lookup specific ANN index and perform query there * @return List of approximate nearest neighbour ids with distance from the query embedding. */ override def queryWithDistance( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P, key: Option[String] ): Future[List[NeighborWithDistance[T, D]]] = { val mapping = queryableRef.get() if (!mapping.contains(key)) { Future.value(List()) } else { mapping.get(key).get.queryWithDistance(embedding, numOfNeighbors, runtimeParams) } } private def getGroupMapping(): Map[Option[String], Queryable[T, P, D]] = { val groupDirs = indexPathProvider.provideIndexPathWithGroups(rootDir).get() val mapping = groupDirs.map { groupDir => val queryable = buildIndex(groupDir) Option(groupDir.getName) -> queryable }.toMap mapping } /** * ANN query for ids. * * @param embedding : Embedding/Vector to be queried with. * @param numOfNeighbors : Number of neighbours to be queried for. * @param runtimeParams : Runtime params associated with index to control accuracy/latency etc. * * @return List of approximate nearest neighbour ids. */ override def query( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P ): Future[List[T]] = { query(embedding, numOfNeighbors, runtimeParams, None) } /** * ANN query for ids with distance. * * @param embedding : Embedding/Vector to be queried with. * @param numOfNeighbors : Number of neighbours to be queried for. * @param runtimeParams : Runtime params associated with index to control accuracy/latency etc. * * @return List of approximate nearest neighbour ids with distance from the query embedding. */ override def queryWithDistance( embedding: EmbeddingVector, numOfNeighbors: Int, runtimeParams: P ): Future[List[NeighborWithDistance[T, D]]] = { queryWithDistance(embedding, numOfNeighbors, runtimeParams, None) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/UnsafeQueryIndexServer.scala ================================================ package com.twitter.ann.service.query_server.common import com.google.common.util.concurrent.MoreExecutors import com.google.inject.Module import com.twitter.ann.common._ import com.twitter.ann.common.thriftscala.{Distance => ServiceDistance} import com.twitter.ann.common.thriftscala.{RuntimeParams => ServiceRuntimeParams} import com.twitter.app.Flag import com.twitter.bijection.Injection import com.twitter.cortex.ml.embeddings.common.EntityKind import com.twitter.finatra.thrift.routing.ThriftRouter import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.ThreadPoolExecutor import java.util.concurrent.TimeUnit /** * This class is used when you do not know the generic parameters of the Server at compile time. * If you want compile time checks that your parameters are correct use QueryIndexServer instead. * In particular, when you want to have these id, distance and the runtime params as cli options you * should extend this class. */ abstract class UnsafeQueryIndexServer[R <: RuntimeParams] extends BaseQueryIndexServer { private[this] val metricName = flag[String]("metric", "metric") private[this] val idType = flag[String]("id_type", "type of ids to use") private[query_server] val queryThreads = flag[Int]( "threads", System .getProperty("mesos.resources.cpu", s"${Runtime.getRuntime.availableProcessors()}").toInt, "Size of thread pool for concurrent querying" ) private[query_server] val dimension = flag[Int]("dimension", "dimension") private[query_server] val indexDirectory = flag[String]("index_directory", "index directory") private[query_server] val refreshable = flag[Boolean]("refreshable", false, "if index is refreshable or not") private[query_server] val refreshableInterval = flag[Int]("refreshable_interval_minutes", 10, "refreshable index update interval") private[query_server] val sharded = flag[Boolean]("sharded", false, "if index is sharded") private[query_server] val shardedHours = flag[Int]("sharded_hours", "how many shards load at once") private[query_server] val shardedWatchLookbackIndexes = flag[Int]("sharded_watch_lookback_indexes", "how many indexes backwards to watch") private[query_server] val shardedWatchIntervalMinutes = flag[Int]("sharded_watch_interval_minutes", "interval at which hdfs is watched for changes") private[query_server] val minIndexSizeBytes = flag[Long]("min_index_size_byte", 0, "min index size in bytes") private[query_server] val maxIndexSizeBytes = flag[Long]("max_index_size_byte", Long.MaxValue, "max index size in bytes") private[query_server] val grouped = flag[Boolean]("grouped", false, "if indexes are grouped") private[query_server] val qualityFactorEnabled = flag[Boolean]( "quality_factor_enabled", false, "Enable dynamically reducing search complexity when cgroups container is throttled. Useful to disable when load testing" ) private[query_server] val warmup_enabled: Flag[Boolean] = flag("warmup", false, "Enable warmup before the query server starts up") // Time to wait for the executor to finish before terminating the JVM private[this] val terminationTimeoutMs = 100 protected lazy val executor: ExecutorService = MoreExecutors.getExitingExecutorService( Executors.newFixedThreadPool(queryThreads()).asInstanceOf[ThreadPoolExecutor], terminationTimeoutMs, TimeUnit.MILLISECONDS ) protected lazy val unsafeMetric: Metric[_] with Injection[_, ServiceDistance] = { Metric.fromString(metricName()) } override protected val additionalModules: Seq[Module] = Seq() override final def addController(router: ThriftRouter): Unit = { router.add(queryIndexThriftController) } protected def unsafeQueryableMap[T, D <: Distance[D]]: Queryable[T, R, D] protected val runtimeInjection: Injection[R, ServiceRuntimeParams] private[this] def queryIndexThriftController[ T, D <: Distance[D] ]: QueryIndexThriftController[T, R, D] = { val controller = new QueryIndexThriftController[T, R, D]( statsReceiver.scope("ann_server"), unsafeQueryableMap.asInstanceOf[Queryable[T, R, D]], runtimeInjection, unsafeMetric.asInstanceOf[Injection[D, ServiceDistance]], idInjection[T]() ) logger.info("QueryIndexThriftController created....") controller } protected final def idInjection[T](): Injection[T, Array[Byte]] = { val idInjection = idType() match { case "string" => AnnInjections.StringInjection case "long" => AnnInjections.LongInjection case "int" => AnnInjections.IntInjection case entityKind => EntityKind.getEntityKind(entityKind).byteInjection } idInjection.asInstanceOf[Injection[T, Array[Byte]]] } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/throttling/AuroraCPUStatsReader.scala ================================================ package com.twitter.ann.service.query_server.common.throttling import com.twitter.server.filter.CgroupsCpu class AuroraCPUStatsReader() { val cgroupsCpu = new CgroupsCpu() def throttledTimeNanos(): Option[Long] = cgroupsCpu.cpuStat.map { cs => cs.throttledTimeNanos } /** * Read assigned cpu number from Mesos files * * @return positive number is the number of CPUs (can be fractional). * -1 means file read failed or it's not a valid Mesos environment. */ def cpuQuota: Double = cgroupsCpu.cfsPeriodMicros match { case -1L => -1.0 case 0L => 0.0 // avoid divide by 0 case periodMicros => cgroupsCpu.cfsQuotaMicros match { case -1L => -1.0 case quotaMicros => quotaMicros.toDouble / periodMicros.toDouble } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/throttling/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], tags = ["bazel-compatible"], dependencies = [ "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/scala/com/twitter/ann/faiss", "ann/src/main/scala/com/twitter/ann/hnsw", "twitter-server-internal/src/main/scala", "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/throttling/ThrottlingBasedQualityTask.scala ================================================ package com.twitter.ann.service.query_server.common.throttling import com.twitter.ann.common.RuntimeParams import com.twitter.ann.common.Task import com.twitter.ann.faiss.FaissParams import com.twitter.ann.hnsw.HnswParams import com.twitter.ann.service.query_server.common.throttling.ThrottlingBasedQualityTask.SAMPLING_INTERVAL import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.finagle.stats.StatsReceiver import com.twitter.util.Duration import com.twitter.util.Future import com.twitter.util.logging.Logging object ThrottlingBasedQualityTask { private[throttling] val SAMPLING_INTERVAL = 100.milliseconds } class ThrottlingBasedQualityTask( override val statsReceiver: StatsReceiver, // Parameters are taken from OverloadAdmissionController instrument: ThrottlingInstrument = new WindowedThrottlingInstrument(SAMPLING_INTERVAL, 5, new AuroraCPUStatsReader())) extends Task with Logging { import ThrottlingBasedQualityTask._ // [0, 1] where 1 is fully throttled // Quickly throttle, but dampen recovery to make sure we won't enter throttle/GC death spiral @volatile private var dampenedThrottlingPercentage: Double = 0 protected[throttling] def task(): Future[Unit] = { if (!instrument.disabled) { instrument.sample() val delta = instrument.percentageOfTimeSpentThrottling - dampenedThrottlingPercentage if (delta > 0) { // We want to start shedding load, do it quickly dampenedThrottlingPercentage += delta } else { // Recover much slower // At the rate of 100ms per sample, lookback is 2 minutes val samplesToConverge = 1200.toDouble dampenedThrottlingPercentage = dampenedThrottlingPercentage + delta * (2 / (samplesToConverge + 1)) } statsReceiver.stat("dampened_throttling").add(dampenedThrottlingPercentage.toFloat * 100) } Future.Unit } protected def taskInterval: Duration = SAMPLING_INTERVAL def discountParams[T <: RuntimeParams](params: T): T = { // [0, 1] where 1 is best quality and lowest speed // It's expected to run @1 majority of time val qualityFactor = math.min(1, math.max(0, 1 - dampenedThrottlingPercentage)) def applyQualityFactor(param: Int) = math.max(1, math.ceil(param * qualityFactor).toInt) params match { case HnswParams(ef) => HnswParams(applyQualityFactor(ef)).asInstanceOf[T] case FaissParams(nprobe, quantizerEf, quantizerKFactorRF, quantizerNprobe, ht) => FaissParams( nprobe.map(applyQualityFactor), quantizerEf.map(applyQualityFactor), quantizerKFactorRF.map(applyQualityFactor), quantizerNprobe.map(applyQualityFactor), ht.map(applyQualityFactor) ).asInstanceOf[T] } } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/throttling/WindowedStats.scala ================================================ package com.twitter.ann.service.query_server.common.throttling /** * A simple ring buffer that keeps track of long values over `window`. */ private[throttling] class WindowedStats(window: Int) { private[this] val buffer = new Array[Long](window) private[this] var index = 0 private[this] var sumValue = 0L private[this] var count = 0 def add(v: Long): Unit = { count = math.min(count + 1, window) val old = buffer(index) buffer(index) = v index = (index + 1) % window sumValue += v - old } def avg: Double = { sumValue.toDouble / count } def sum: Long = { sumValue } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/throttling/WindowedThrottlingInstrument.scala ================================================ package com.twitter.ann.service.query_server.common.throttling import com.twitter.util.Duration trait ThrottlingInstrument { def sample(): Unit def percentageOfTimeSpentThrottling(): Double def disabled: Boolean } class WindowedThrottlingInstrument( stepFrequency: Duration, windowLengthInFrequencySteps: Int, reader: AuroraCPUStatsReader) extends ThrottlingInstrument { private[this] val throttlingChangeHistory: WindowedStats = new WindowedStats( windowLengthInFrequencySteps) private[this] val cpuQuota: Double = reader.cpuQuota // The total number of allotted CPU time per step (in nanos). private[this] val assignedCpu: Duration = stepFrequency * cpuQuota private[this] val assignedCpuNs: Long = assignedCpu.inNanoseconds @volatile private[this] var previousThrottledTimeNs: Long = 0 /** * If there isn't a limit on how much cpu the container can use, aurora * throttling will never kick in. */ final def disabled: Boolean = cpuQuota <= 0 def sample(): Unit = sampleThrottling() match { case Some(load) => throttlingChangeHistory.add(load) case None => () } private[this] def sampleThrottling(): Option[Long] = reader.throttledTimeNanos().map { throttledTimeNs => val throttlingChange = throttledTimeNs - previousThrottledTimeNs previousThrottledTimeNs = throttledTimeNs throttlingChange } // Time spent throttling over windowLength, normalized by number of CPUs def percentageOfTimeSpentThrottling(): Double = { math.min(1, throttlingChangeHistory.sum.toDouble / assignedCpuNs) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/warmup/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], tags = ["bazel-compatible"], dependencies = [ "ann/src/main/scala/com/twitter/ann/common", "util/util-core:scala", "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/common/warmup/Warmup.scala ================================================ package com.twitter.ann.service.query_server.common.warmup import com.twitter.ann.common.EmbeddingType.EmbeddingVector import com.twitter.ml.api.embedding.Embedding import com.twitter.util.Await import com.twitter.util.Duration import com.twitter.util.Future import com.twitter.util.Return import com.twitter.util.Throw import com.twitter.util.Try import com.twitter.util.logging.Logging import scala.annotation.tailrec import scala.util.Random trait Warmup extends Logging { protected def minSuccessfulTries: Int protected def maxTries: Int protected def randomQueryDimension: Int protected def timeout: Duration @tailrec final protected def run( iteration: Int = 0, successes: Int = 0, name: String, f: => Future[_] ): Unit = { if (successes == minSuccessfulTries || iteration == maxTries) { info(s"Warmup finished after ${iteration} iterations with ${successes} successes") } else { Try(Await.result(f.liftToTry, timeout)) match { case Return(Return(_)) => debug(s"[$name] Iteration $iteration Success") run(iteration + 1, successes + 1, name, f) case Return(Throw(e)) => warn(s"[$name] Iteration $iteration has failed: ${e.getMessage}. ", e) run(iteration + 1, successes, name, f) case Throw(e) => info(s"[$name] Iteration $iteration was too slow: ${e.getMessage}. ", e) run(iteration + 1, successes, name, f) } } } private val rng = new Random() protected def randomQuery(): EmbeddingVector = Embedding(Array.fill(randomQueryDimension)(-1 + 2 * rng.nextFloat())) def warmup(): Unit } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/faiss/BUILD ================================================ scala_library( name = "server", sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], tags = ["bazel-compatible"], dependencies = [ "ann/src/main/java/com/twitter/ann/faiss", "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/scala/com/twitter/ann/faiss", "ann/src/main/scala/com/twitter/ann/service/query_server/common", "ann/src/main/scala/com/twitter/ann/service/query_server/common:faiss_index_path_provider", "ann/src/main/scala/com/twitter/ann/service/query_server/common/throttling", "ann/src/main/scala/com/twitter/ann/service/query_server/common/warmup", "src/java/com/twitter/search/common/file", ], ) jvm_binary( name = "faiss-query-server", main = "com.twitter.ann.service.query_server.faiss.FaissQueryIndexServer", compiler_option_sets = ["fatal_warnings"], runtime_platform = "java11", tags = ["bazel-compatible"], dependencies = [ ":server", "3rdparty/jvm/ch/qos/logback:logback-classic", "3rdparty/jvm/org/slf4j:jcl-over-slf4j", "3rdparty/jvm/org/slf4j:jul-to-slf4j", "3rdparty/jvm/org/slf4j:log4j-over-slf4j", "ann/src/main/resources", "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/faiss/FaissQueryIndexServer.scala ================================================ package com.twitter.ann.service.query_server.faiss import com.twitter.ann.common.Distance import com.twitter.ann.common.QueryableOperations.Map import com.twitter.ann.common._ import com.twitter.ann.common.thriftscala.{RuntimeParams => ServiceRuntimeParams} import com.twitter.ann.faiss.FaissCommon import com.twitter.ann.faiss.FaissIndex import com.twitter.ann.faiss.FaissParams import com.twitter.ann.faiss.HourlyShardedIndex import com.twitter.ann.service.query_server.common.QueryableProvider import com.twitter.ann.service.query_server.common.RefreshableQueryable import com.twitter.ann.service.query_server.common.UnsafeQueryIndexServer import com.twitter.ann.service.query_server.common.FaissIndexPathProvider import com.twitter.ann.service.query_server.common.throttling.ThrottlingBasedQualityTask import com.twitter.ann.service.query_server.common.warmup.Warmup import com.twitter.bijection.Injection import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.search.common.file.AbstractFile import com.twitter.search.common.file.FileUtils import com.twitter.util.Duration import java.util.concurrent.TimeUnit object FaissQueryIndexServer extends FaissQueryableServer class FaissQueryableServer extends UnsafeQueryIndexServer[FaissParams] { // given a directory, how to load it as a queryable index def queryableProvider[T, D <: Distance[D]]: QueryableProvider[T, FaissParams, D] = new QueryableProvider[T, FaissParams, D] { override def provideQueryable( directory: AbstractFile ): Queryable[T, FaissParams, D] = { FaissIndex.loadIndex[T, D]( dimension(), unsafeMetric.asInstanceOf[Metric[D]], directory ) } } private def buildSimpleQueryable[T, D <: Distance[D]]( dir: AbstractFile ): Queryable[T, FaissParams, D] = { val queryable = if (refreshable()) { logger.info(s"build refreshable queryable") val updatableQueryable = new RefreshableQueryable( false, dir, queryableProvider.asInstanceOf[QueryableProvider[T, FaissParams, D]], FaissIndexPathProvider( minIndexSizeBytes(), maxIndexSizeBytes(), statsReceiver.scope("validated_index_provider") ), statsReceiver.scope("refreshable_queryable"), updateInterval = refreshableInterval().minutes ) // init first load of index and also schedule the following reloads updatableQueryable.start() updatableQueryable.asInstanceOf[QueryableGrouped[T, FaissParams, D]] } else { logger.info(s"build non-refreshable queryable") logger.info(s"Loading ${dir}") queryableProvider.provideQueryable(dir).asInstanceOf[Queryable[T, FaissParams, D]] } logger.info("Faiss queryable created....") queryable } private def buildShardedQueryable[T, D <: Distance[D]]( dir: AbstractFile ): Queryable[T, FaissParams, D] = { logger.info(s"build sharded queryable") val queryable = HourlyShardedIndex.loadIndex[T, D]( dimension(), unsafeMetric.asInstanceOf[Metric[D]], dir, shardedHours(), Duration(shardedWatchIntervalMinutes(), TimeUnit.MINUTES), shardedWatchLookbackIndexes(), statsReceiver.scope("hourly_sharded_index") ) logger.info("Faiss sharded queryable created....") closeOnExit(queryable) queryable.startImmediately() logger.info("Directory watching is scheduled") queryable } // Readings come incorrect if reader is created too early in the lifecycle of a server // hence lazy private lazy val throttleSamplingTask = new ThrottlingBasedQualityTask( statsReceiver.scope("throttling_task")) override def unsafeQueryableMap[T, D <: Distance[D]]: Queryable[T, FaissParams, D] = { val dir = FileUtils.getFileHandle(indexDirectory()) val queryable = if (sharded()) { require(shardedHours() > 0, "Number of hourly shards must be specified") require(shardedWatchIntervalMinutes() > 0, "Shard watch interval must be specified") require(shardedWatchLookbackIndexes() > 0, "Index lookback must be specified") buildShardedQueryable[T, D](dir) } else { buildSimpleQueryable[T, D](dir) } if (qualityFactorEnabled()) { logger.info("Quality Factor throttling is enabled") closeOnExit(throttleSamplingTask) throttleSamplingTask.jitteredStart() queryable.mapRuntimeParameters(throttleSamplingTask.discountParams) } else { queryable } } override val runtimeInjection: Injection[FaissParams, ServiceRuntimeParams] = FaissCommon.RuntimeParamsInjection protected override def warmup(): Unit = if (warmup_enabled()) new FaissWarmup(unsafeQueryableMap, dimension()).warmup() } class FaissWarmup(faiss: Queryable[_, FaissParams, _], dimension: Int) extends Warmup { protected def minSuccessfulTries: Int = 100 protected def maxTries: Int = 1000 protected def timeout: Duration = 50.milliseconds protected def randomQueryDimension: Int = dimension def warmup(): Unit = { run( name = "queryWithDistance", f = faiss .queryWithDistance( randomQuery(), 100, FaissParams(nprobe = Some(128), None, None, None, None)) ) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/hnsw/BUILD ================================================ scala_library( name = "server", sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], tags = ["bazel-compatible"], dependencies = [ "ann/src/main/scala/com/twitter/ann/common", "ann/src/main/scala/com/twitter/ann/hnsw", "ann/src/main/scala/com/twitter/ann/service/query_server/common", "ann/src/main/scala/com/twitter/ann/service/query_server/common/warmup", "src/java/com/twitter/search/common/file", ], ) jvm_binary( name = "hnsw-query-server", main = "com.twitter.ann.service.query_server.hnsw.HnswQueryIndexServer", compiler_option_sets = ["fatal_warnings"], runtime_platform = "java11", tags = [ "bazel-compatible", ], dependencies = [ ":server", "3rdparty/jvm/ch/qos/logback:logback-classic", "3rdparty/jvm/org/slf4j:jcl-over-slf4j", "3rdparty/jvm/org/slf4j:jul-to-slf4j", "3rdparty/jvm/org/slf4j:log4j-over-slf4j", "ann/src/main/resources", "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/service/query_server/hnsw/HnswQueryIndexServer.scala ================================================ package com.twitter.ann.service.query_server.hnsw import com.twitter.ann.common.Distance import com.twitter.ann.common._ import com.twitter.ann.common.thriftscala.{RuntimeParams => ServiceRuntimeParams} import com.twitter.ann.hnsw.HnswCommon import com.twitter.ann.hnsw.HnswParams import com.twitter.ann.hnsw.TypedHnswIndex import com.twitter.ann.service.query_server.common.QueryableProvider import com.twitter.ann.service.query_server.common.RefreshableQueryable import com.twitter.ann.service.query_server.common.UnsafeQueryIndexServer import com.twitter.ann.service.query_server.common.ValidatedIndexPathProvider import com.twitter.ann.service.query_server.common.warmup.Warmup import com.twitter.bijection.Injection import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.search.common.file.AbstractFile import com.twitter.search.common.file.FileUtils import com.twitter.util.Duration import com.twitter.util.FuturePool // Creating a separate hnsw query server object, since unit test require non singleton server. object HnswQueryIndexServer extends HnswQueryableServer class HnswQueryableServer extends UnsafeQueryIndexServer[HnswParams] { private val IndexGroupPrefix = "group_" // given a directory, how to load it as a queryable index def queryableProvider[T, D <: Distance[D]]: QueryableProvider[T, HnswParams, D] = new QueryableProvider[T, HnswParams, D] { override def provideQueryable( dir: AbstractFile ): Queryable[T, HnswParams, D] = { TypedHnswIndex.loadIndex[T, D]( dimension(), unsafeMetric.asInstanceOf[Metric[D]], idInjection[T](), ReadWriteFuturePool(FuturePool.interruptible(executor)), dir ) } } private def buildQueryable[T, D <: Distance[D]]( dir: AbstractFile, grouped: Boolean ): Queryable[T, HnswParams, D] = { val queryable = if (refreshable()) { logger.info(s"build refreshable queryable") val updatableQueryable = new RefreshableQueryable( grouped, dir, queryableProvider.asInstanceOf[QueryableProvider[T, HnswParams, D]], ValidatedIndexPathProvider( minIndexSizeBytes(), maxIndexSizeBytes(), statsReceiver.scope("validated_index_provider") ), statsReceiver.scope("refreshable_queryable"), updateInterval = refreshableInterval().minutes ) // init first load of index and also schedule the following reloads updatableQueryable.start() updatableQueryable.asInstanceOf[QueryableGrouped[T, HnswParams, D]] } else { logger.info(s"build non-refreshable queryable") queryableProvider.provideQueryable(dir).asInstanceOf[Queryable[T, HnswParams, D]] } logger.info("Hnsw queryable created....") queryable } override def unsafeQueryableMap[T, D <: Distance[D]]: Queryable[T, HnswParams, D] = { val dir = FileUtils.getFileHandle(indexDirectory()) buildQueryable(dir, grouped()) } override val runtimeInjection: Injection[HnswParams, ServiceRuntimeParams] = HnswCommon.RuntimeParamsInjection protected override def warmup(): Unit = if (warmup_enabled()) new HNSWWarmup(unsafeQueryableMap, dimension()).warmup() } class HNSWWarmup(hnsw: Queryable[_, HnswParams, _], dimension: Int) extends Warmup { protected def minSuccessfulTries: Int = 100 protected def maxTries: Int = 1000 protected def timeout: Duration = 50.milliseconds protected def randomQueryDimension: Int = dimension def warmup(): Unit = { run( name = "queryWithDistance", f = hnsw .queryWithDistance(randomQuery(), 100, HnswParams(ef = 800)) ) } } ================================================ FILE: ann/src/main/scala/com/twitter/ann/util/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "ann/src/main/scala/com/twitter/ann/common", "util/util-logging", ], ) ================================================ FILE: ann/src/main/scala/com/twitter/ann/util/IndexBuilderUtils.scala ================================================ package com.twitter.ann.util import com.twitter.ann.common.{Appendable, EntityEmbedding} import com.twitter.concurrent.AsyncStream import com.twitter.logging.Logger import com.twitter.util.Future import java.util.concurrent.atomic.AtomicInteger object IndexBuilderUtils { val Log = Logger.apply() def addToIndex[T]( appendable: Appendable[T, _, _], embeddings: Seq[EntityEmbedding[T]], concurrencyLevel: Int ): Future[Int] = { val count = new AtomicInteger() // Async stream allows us to procss at most concurrentLevel futures at a time. Future.Unit.before { val stream = AsyncStream.fromSeq(embeddings) val appendStream = stream.mapConcurrent(concurrencyLevel) { annEmbedding => val processed = count.incrementAndGet() if (processed % 10000 == 0) { Log.info(s"Performed $processed updates") } appendable.append(annEmbedding) } appendStream.size } } } ================================================ FILE: ann/src/main/thrift/com/twitter/ann/common/BUILD ================================================ create_thrift_libraries( base_name = "ann-common", sources = ["*.thrift"], platform = "java8", tags = ["bazel-compatible"], dependency_roots = [ "mediaservices/commons/src/main/thrift", "src/thrift/com/twitter/ml/api:embedding", ], generate_languages = [ "java", "python", "scala", "strato", ], provides_java_name = "ann-common-thrift-java", provides_scala_name = "ann-common-thrift-scala", ) ================================================ FILE: ann/src/main/thrift/com/twitter/ann/common/ann_common.thrift ================================================ namespace java com.twitter.ann.common.thriftjava #@namespace scala com.twitter.ann.common.thriftscala #@namespace strato com.twitter.ann.common namespace py gen.twitter.ann.common include "com/twitter/mediaservices/commons/ServerCommon.thrift" include "com/twitter/ml/api/embedding.thrift" /** * Thrift schema for storing file based Index mapping */ struct FileBasedIndexIdStore { 1: optional map indexIdMap } enum DistanceMetric { L2, Cosine, InnerProduct, RESERVED_4, RESERVED_5, RESERVED_6, RESERVED_7, EditDistance } (persisted = 'true', strato.graphql.typename='DistanceMetric') struct AnnoyIndexMetadata { 1: i32 dimension 2: DistanceMetric distanceMetric 3: i32 numOfTrees 4: i64 numOfVectorsIndexed } (persisted = 'true', strato.graphql.typename='AnnoyIndexMetadata') struct AnnoyRuntimeParam { /* Number of vectors to evaluate while searching. A larger value will give more accurate results, but will take longer time to return. * Default value would be numberOfTrees*numberOfNeigboursRequested */ 1: optional i32 numOfNodesToExplore } struct HnswRuntimeParam { // More the value of ef better the recall with but at cost of latency. // Set it greater than equal to number of neighbours required. 1: i32 ef } // These options are subset of all possible parameters, defined by // https://github.com/facebookresearch/faiss/blob/36f2998a6469280cef3b0afcde2036935a29aa1f/faiss/AutoTune.cpp#L444 // quantizer_ prefix changes IndexIVF.quantizer parameters instead struct FaissRuntimeParam { // How many cells to visit in IVFPQ. Higher is slower / more precise. 1: optional i32 nprobe // Depth of search in HNSW. Higher is slower / more precise. 2: optional i32 quantizer_ef // How many times more neighbours are requested from underlying index by IndexRefine. 3: optional i32 quantizer_kfactor_rf // Same as 1: but for quantizer 4: optional i32 quantizer_nprobe // Hamming distance threshold to filter neighbours when searching. 5: optional i32 ht } // Every ANN index will have this metadata and it'll be used by the query service for validation. struct AnnIndexMetadata { 1: optional i64 timestamp 2: optional i32 index_size 3: optional bool withGroups 4: optional i32 numGroups } (persisted = 'true') struct HnswIndexMetadata { 1: i32 dimension 2: DistanceMetric distanceMetric 3: i32 numElements } (persisted = 'true') struct HnswInternalIndexMetadata { 1: i32 maxLevel 2: optional binary entryPoint 3: i32 efConstruction 4: i32 maxM 5: i32 numElements } (persisted = 'true') struct HnswGraphEntry { 1: i32 level 2: binary key 3: list neighbours } (persisted = 'true', strato.graphql.typename='HnswGraphEntry') enum IndexType { TWEET, USER, WORD, LONG, INT, STRING, RESERVED_7, RESERVED_8, RESERVED_9, RESERVED_10 } (persisted = 'true', strato.graphql.typename='IndexType') struct CosineDistance { 1: required double distance } struct L2Distance { 1: required double distance } struct InnerProductDistance { 1: required double distance } struct EditDistance { 1: required i32 distance } union Distance { 1: CosineDistance cosineDistance 2: L2Distance l2Distance 3: InnerProductDistance innerProductDistance 4: EditDistance editDistance } struct NearestNeighbor { 1: required binary id 2: optional Distance distance } struct NearestNeighborResult { // This list is ordered from nearest to furthest neighbor 1: required list nearestNeighbors } // Different runtime/tuning params while querying for indexes to control accuracy/latency etc.. union RuntimeParams { 1: AnnoyRuntimeParam annoyParam 2: HnswRuntimeParam hnswParam 3: FaissRuntimeParam faissParam } struct NearestNeighborQuery { 1: required embedding.Embedding embedding 2: required bool with_distance 3: required RuntimeParams runtimeParams, 4: required i32 numberOfNeighbors, // The purpose of the key here is to load the index in memory as a map of Option[key] to index // If the key is not specified in the query, the map value corresponding to None key will be used // as the queryable index to perform Nearest Neighbor search on 5: optional string key } enum BadRequestCode { VECTOR_DIMENSION_MISMATCH, RESERVED_2, RESERVED_3, RESERVED_4, RESERVED_5, RESERVED_6, RESERVED_7, RESERVED_8, RESERVED_9 } exception BadRequest { 1: string message 2: required BadRequestCode code } service AnnQueryService { /** * Get approximate nearest neighbor for a given vector */ NearestNeighborResult query(1: NearestNeighborQuery query) throws (1: ServerCommon.ServerError serverError, 2: BadRequest badRequest) } ================================================ FILE: ann/src/main/thrift/com/twitter/ann/knn/BUILD ================================================ create_thrift_libraries( base_name = "thrift", sources = ["*.thrift"], platform = "java8", tags = ["bazel-compatible"], dependency_roots = ["src/thrift/com/twitter/ml/featurestore:ml-feature-store"], generate_languages = [ "java", "scala", ], ) ================================================ FILE: ann/src/main/thrift/com/twitter/ann/knn/knn.thrift ================================================ namespace java com.twitter.ann.knn.thriftjava #@namespace scala com.twitter.ann.knn.thriftscala namespace py gen.twitter.ann.knn include "com/twitter/ml/featurestore/entity.thrift" struct Neighbor { 1: required double distance 2: required entity.EntityId id } (persisted = "true") struct Knn { 1: required entity.EntityId queryId 2: required list neighbors }(persisted='true') ================================================ FILE: ann/src/main/thrift/com/twitter/ann/serialization/BUILD ================================================ create_thrift_libraries( base_name = "serialization", sources = ["*.thrift"], platform = "java8", tags = ["bazel-compatible"], dependency_roots = [ "src/thrift/com/twitter/ml/api:embedding", ], generate_languages = [ "java", "scala", ], ) ================================================ FILE: ann/src/main/thrift/com/twitter/ann/serialization/serialization.thrift ================================================ #@namespace scala com.twitter.ann.serialization.thriftscala include "com/twitter/ml/api/embedding.thrift" /** * Thrift schema for storing embeddings in a file */ struct PersistedEmbedding { 1: required binary id 2: required embedding.Embedding embedding }(persisted = 'true') ================================================ FILE: ci/ci.sh ================================================ #!/bin/sh exit 0 ================================================ FILE: cr-mixer/BUILD.bazel ================================================ jvm_binary( name = "bin", basename = "cr-mixer", main = "com.twitter.cr_mixer.CrMixerServerMain", runtime_platform = "java11", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/ch/qos/logback:logback-classic", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer", "finagle/finagle-zipkin-scribe/src/main/scala", "finatra/inject/inject-logback/src/main/scala", "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", "twitter-server-internal/src/main/scala", "twitter-server/logback-classic/src/main/scala", ], ) # Aurora Workflows build phase convention requires a jvm_app named with ${project-name}-app jvm_app( name = "cr-mixer-app", archive = "zip", binary = ":bin", tags = ["bazel-compatible"], ) ================================================ FILE: cr-mixer/README.md ================================================ # CR-Mixer CR-Mixer is a candidate generation service proposed as part of the Personalization Strategy vision for Twitter. Its aim is to speed up the iteration and development of candidate generation and light ranking. The service acts as a lightweight coordinating layer that delegates candidate generation tasks to underlying compute services. It focuses on Twitter's candidate generation use cases and offers a centralized platform for fetching, mixing, and managing candidate sources and light rankers. The overarching goal is to increase the speed and ease of testing and developing candidate generation pipelines, ultimately delivering more value to Twitter users. CR-Mixer acts as a configurator and delegator, providing abstractions for the challenging parts of candidate generation and handling performance issues. It will offer a 1-stop-shop for fetching and mixing candidate sources, a managed and shared performant platform, a light ranking layer, a common filtering layer, a version control system, a co-owned feature switch set, and peripheral tooling. CR-Mixer's pipeline consists of 4 steps: source signal extraction, candidate generation, filtering, and ranking. It also provides peripheral tooling like scribing, debugging, and monitoring. The service fetches source signals externally from stores like UserProfileService and RealGraph, calls external candidate generation services, and caches results. Filters are applied for deduping and pre-ranking, and a light ranking step follows. ================================================ FILE: cr-mixer/server/src/main/resources/BUILD.bazel ================================================ resources( sources = [ "*.xml", "*.yml", "config/*.yml", ], tags = ["bazel-compatible"], ) ================================================ FILE: cr-mixer/server/src/main/resources/config/decider.yml ================================================ # The keys in this file correspond to the DeciderValues defined in # https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/DeciderKey.scala dark_traffic_filter: comment: Proportion of the requests that are forwarded as dark traffic to the proxy default_availability: 0 enable_tweet_recommendations_home_product: comment: Proportion of requests where we return an actual response for TweetRecommendations Home product default_availability: 10000 enable_tweet_health_score: comment: "Enable the calculation for health scores in tweetInfo. By enabling this decider, we will compute TweetHealthModelScore" default_availability: 10000 enable_user_agatha_score: comment: "Enable the calculation for health scores in tweetInfo. By enabling this decider, we will compute UserHealthModelScore" default_availability: 10000 enable_user_tweet_entity_graph_traffic: comment: "Enable the traffic to user entity tweet graph to fetch liked-by tweets candidates" default_availability: 10000 enable_user_tweet_graph_traffic: comment: "Enable the traffic to user tweet graph to fetch similar tweets candidates" default_availability: 10000 enable_user_video_graph_traffic: comment: "Enable the traffic to user video graph to fetch similar tweets candidates" default_availability: 10000 enable_user_ad_graph_traffic: comment: "Enable the traffic to user ad graph to fetch similar tweets candidates" default_availability: 10000 enable_qig_similar_tweets_traffic: comment: "Enable the traffic to QIG to fetch similar tweet candidates" default_availability: 0 enable_frs_traffic: comment: "Enable the traffic to FRS to fetch user follow recommendations" default_availability: 0 enable_hydra_dark_traffic: comment: "Enable dark traffic to hydra" default_availability: 0 enable_real_graph_mh_store: comment: "Enable traffic for the real graph manhattan based store" default_availability: 0 enable_simclusters_ann_experimental_dark_traffic: comment: "Enable dark traffic to simclusters-ann-experimental" default_availability: 0 enable_simclusters_ann_2_dark_traffic: comment: "Enable dark traffic to prod SimClustersANN2" default_availability: 0 enable_user_state_store: comment: "Enable traffic user state store to hydrate user state" default_availability: 0 upper_funnel_per_step_scribe_rate: comment: "Enable Upper Funnel Event Scribe Sampling (fetch, pre-rank, interleave etc.) for getTweetsRecommendations() endpoint" default_availability: 0 kafka_message_scribe_sample_rate: comment: "Gates the production of forked scribe messages to kafka for the async feature hydrator" default_availability: 0 top_level_api_ddg_metrics_scribe_rate: comment: "Enable Top Level API DDG Metrics Scribe Sampling for getTweetsRecommendations() endpoint" default_availability: 0 ads_recommendations_per_experiment_scribe_rate: comment: "Percentage of DDG traffic to Scribe for getAdsRecommendations() endpoint" default_availability: 0 enable_loadshedding_getTweetRecommendations: comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response" default_availability: 0 enable_loadshedding_getTweetRecommendations_Home: comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response" default_availability: 0 enable_loadshedding_getTweetRecommendations_Notifications: comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response" default_availability: 0 enable_loadshedding_getTweetRecommendations_Email: comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response" default_availability: 0 enable_loadshedding_getRelatedTweetsForQueryTweet: comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response" default_availability: 0 enable_loadshedding_getRelatedTweetsForQueryTweet_Home: comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response" default_availability: 0 enable_loadshedding_getRelatedTweetsForQueryTweet_MoreTweetsModule: comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response" default_availability: 0 enable_loadshedding_getRelatedTweetsForQueryAuthor: comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response" default_availability: 0 enable_loadshedding_getRelatedTweetsForQueryAuthor_MoreTweetsModule: comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response" default_availability: 0 enable_loadshedding_getFrsBasedTweetRecommendations_Home: comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response" default_availability: 0 enable_loadshedding_getFrsBasedTweetRecommendations_Notifications: comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response" default_availability: 0 enable_user_media_representation_store: comment: "Enable fetching user nudity rate signal from Media Understanding" default_availability: 0 enable_magic_recs_real_time_aggregates_store: comment: "Enable fetching real time aggregates features from Magic Recs memcache" default_availability: 0 enable_utg_realtime_tweet_engagement_score: comment: "Enable fetching real time tweet engagement score from utg-plus" default_availability: 0 get_tweet_recommendations_cache_rate: comment: "Proportion of users where getTweetRecommendations() request and responses will be cached" default_availability: 1000 enable_earlybird_traffic: comment: "Enable fetching tweet candidates from Earlybird" default_availability: 0 enable_scribe_for_blue_verified_tweet_candidates: comment: "Enable scribing for tweet candidates from Blue Verified users" default_availability: 0 ================================================ FILE: cr-mixer/server/src/main/resources/logback.xml ================================================ true ${log.service.output} ${log.service.output}.%d.gz 21 true %date %.-3level ${DEFAULT_SERVICE_PATTERN}%n ${log.access.output} ${log.access.output}.%d.gz 21 true ${DEFAULT_ACCESS_PATTERN}%n true ${log.lens.category} ${log.lens.index} ${log.lens.tag}/service %msg true ${log.lens.category} ${log.lens.index} ${log.lens.tag}/access %msg allow_listed_pipeline_executions.log allow_listed_pipeline_executions.log.%d.gz 7 true %date %.-3level ${DEFAULT_SERVICE_PATTERN}%n ${async_queue_size} ${async_max_flush_time} ${async_queue_size} ${async_max_flush_time} ${async_queue_size} ${async_max_flush_time} ${async_queue_size} ${async_max_flush_time} ${async_queue_size} ${async_max_flush_time} ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/javax/inject:javax.inject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "cr-mixer/server/src/main/resources", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/controller", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/featureswitch", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine", "cr-mixer/thrift/src/main/thrift:thrift-scala", "decider/src/main/scala", "finagle/finagle-core/src/main", "finagle/finagle-http/src/main/scala", "finagle/finagle-thriftmux/src/main/scala", "finatra-internal/mtls-http/src/main/scala", "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/http-core/src/main/java/com/twitter/finatra/http", "finatra/inject/inject-app/src/main/scala", "finatra/inject/inject-core/src/main/scala", "finatra/inject/inject-server/src/main/scala", "finatra/inject/inject-utils/src/main/scala", "finatra/utils/src/main/java/com/twitter/finatra/annotations", "hydra/common/libraries/src/main/scala/com/twitter/hydra/common/model_config", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", "relevance-platform/src/main/scala/com/twitter/relevance_platform/common/filters", "src/thrift/com/twitter/timelines/render:thrift-scala", "thrift-web-forms/src/main/scala/com/twitter/thriftwebforms", "thrift-web-forms/src/main/scala/com/twitter/thriftwebforms/view", "timelines/src/main/scala/com/twitter/timelines/features/app", "twitter-server-internal", "twitter-server/server/src/main/scala", "util/util-app/src/main/scala", "util/util-core:scala", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/CrMixerHttpServerWarmupHandler.scala ================================================ package com.twitter.cr_mixer import com.twitter.finatra.http.routing.HttpWarmup import com.twitter.finatra.httpclient.RequestBuilder._ import com.twitter.inject.Logging import com.twitter.inject.utils.Handler import com.twitter.util.Try import javax.inject.Inject import javax.inject.Singleton @Singleton class CrMixerHttpServerWarmupHandler @Inject() (warmup: HttpWarmup) extends Handler with Logging { override def handle(): Unit = { Try(warmup.send(get("/admin/cr-mixer/product-pipelines"), admin = true)()) .onFailure(e => error(e.getMessage, e)) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/CrMixerServer.scala ================================================ package com.twitter.cr_mixer import com.google.inject.Module import com.twitter.cr_mixer.controller.CrMixerThriftController import com.twitter.cr_mixer.featureswitch.SetImpressedBucketsLocalContextFilter import com.twitter.cr_mixer.module.ActivePromotedTweetStoreModule import com.twitter.cr_mixer.module.CertoStratoStoreModule import com.twitter.cr_mixer.module.CrMixerParamConfigModule import com.twitter.cr_mixer.module.EmbeddingStoreModule import com.twitter.cr_mixer.module.FrsStoreModule import com.twitter.cr_mixer.module.MHMtlsParamsModule import com.twitter.cr_mixer.module.OfflineCandidateStoreModule import com.twitter.cr_mixer.module.RealGraphStoreMhModule import com.twitter.cr_mixer.module.RealGraphOonStoreModule import com.twitter.cr_mixer.module.RepresentationManagerModule import com.twitter.cr_mixer.module.RepresentationScorerModule import com.twitter.cr_mixer.module.TweetInfoStoreModule import com.twitter.cr_mixer.module.TweetRecentEngagedUserStoreModule import com.twitter.cr_mixer.module.TweetRecommendationResultsStoreModule import com.twitter.cr_mixer.module.TripCandidateStoreModule import com.twitter.cr_mixer.module.TwhinCollabFilterStratoStoreModule import com.twitter.cr_mixer.module.UserSignalServiceColumnModule import com.twitter.cr_mixer.module.UserSignalServiceStoreModule import com.twitter.cr_mixer.module.UserStateStoreModule import com.twitter.cr_mixer.module.core.ABDeciderModule import com.twitter.cr_mixer.module.core.CrMixerFlagModule import com.twitter.cr_mixer.module.core.CrMixerLoggingABDeciderModule import com.twitter.cr_mixer.module.core.FeatureContextBuilderModule import com.twitter.cr_mixer.module.core.FeatureSwitchesModule import com.twitter.cr_mixer.module.core.KafkaProducerModule import com.twitter.cr_mixer.module.core.LoggerFactoryModule import com.twitter.cr_mixer.module.similarity_engine.ConsumerEmbeddingBasedTripSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.ConsumerEmbeddingBasedTwHINSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.ConsumerEmbeddingBasedTwoTowerSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.ConsumersBasedUserAdGraphSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.ConsumersBasedUserVideoGraphSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.ProducerBasedUserAdGraphSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.ProducerBasedUserTweetGraphSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.ProducerBasedUnifiedSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.SimClustersANNSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.TweetBasedUnifiedSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.TweetBasedQigSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.TweetBasedTwHINSimlarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.TweetBasedUserAdGraphSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.TweetBasedUserTweetGraphSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.TweetBasedUserVideoGraphSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.TwhinCollabFilterLookupSimilarityEngineModule import com.twitter.cr_mixer.module.ConsumersBasedUserAdGraphStoreModule import com.twitter.cr_mixer.module.ConsumersBasedUserTweetGraphStoreModule import com.twitter.cr_mixer.module.ConsumersBasedUserVideoGraphStoreModule import com.twitter.cr_mixer.module.DiffusionStoreModule import com.twitter.cr_mixer.module.EarlybirdRecencyBasedCandidateStoreModule import com.twitter.cr_mixer.module.TwiceClustersMembersStoreModule import com.twitter.cr_mixer.module.StrongTiePredictionStoreModule import com.twitter.cr_mixer.module.thrift_client.AnnQueryServiceClientModule import com.twitter.cr_mixer.module.thrift_client.EarlybirdSearchClientModule import com.twitter.cr_mixer.module.thrift_client.FrsClientModule import com.twitter.cr_mixer.module.thrift_client.QigServiceClientModule import com.twitter.cr_mixer.module.thrift_client.SimClustersAnnServiceClientModule import com.twitter.cr_mixer.module.thrift_client.TweetyPieClientModule import com.twitter.cr_mixer.module.thrift_client.UserTweetGraphClientModule import com.twitter.cr_mixer.module.thrift_client.UserTweetGraphPlusClientModule import com.twitter.cr_mixer.module.thrift_client.UserVideoGraphClientModule import com.twitter.cr_mixer.{thriftscala => st} import com.twitter.finagle.Filter import com.twitter.finatra.annotations.DarkTrafficFilterType import com.twitter.finatra.decider.modules.DeciderModule import com.twitter.finatra.http.HttpServer import com.twitter.finatra.http.routing.HttpRouter import com.twitter.finatra.jackson.modules.ScalaObjectMapperModule import com.twitter.finatra.mtls.http.{Mtls => HttpMtls} import com.twitter.finatra.mtls.thriftmux.Mtls import com.twitter.finatra.mtls.thriftmux.modules.MtlsThriftWebFormsModule import com.twitter.finatra.thrift.ThriftServer import com.twitter.finatra.thrift.filters._ import com.twitter.finatra.thrift.routing.ThriftRouter import com.twitter.hydra.common.model_config.{ConfigModule => HydraConfigModule} import com.twitter.inject.thrift.modules.ThriftClientIdModule import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper import com.twitter.product_mixer.core.module.StratoClientModule import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule import com.twitter.relevance_platform.common.filters.ClientStatsFilter import com.twitter.relevance_platform.common.filters.DarkTrafficFilterModule import com.twitter.cr_mixer.module.SimClustersANNServiceNameToClientMapper import com.twitter.cr_mixer.module.SkitStratoStoreModule import com.twitter.cr_mixer.module.BlueVerifiedAnnotationStoreModule import com.twitter.cr_mixer.module.core.TimeoutConfigModule import com.twitter.cr_mixer.module.grpc_client.NaviGRPCClientModule import com.twitter.cr_mixer.module.similarity_engine.CertoTopicTweetSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.ConsumerBasedWalsSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.DiffusionBasedSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.EarlybirdSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.SkitTopicTweetSimilarityEngineModule import com.twitter.cr_mixer.module.similarity_engine.UserTweetEntityGraphSimilarityEngineModule import com.twitter.cr_mixer.module.thrift_client.HydraPartitionClientModule import com.twitter.cr_mixer.module.thrift_client.HydraRootClientModule import com.twitter.cr_mixer.module.thrift_client.UserAdGraphClientModule import com.twitter.cr_mixer.module.thrift_client.UserTweetEntityGraphClientModule import com.twitter.thriftwebforms.MethodOptions object CrMixerServerMain extends CrMixerServer class CrMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls { override val name = "cr-mixer-server" private val coreModules = Seq( ABDeciderModule, CrMixerFlagModule, CrMixerLoggingABDeciderModule, CrMixerParamConfigModule, new DarkTrafficFilterModule[st.CrMixer.ReqRepServicePerEndpoint](), DeciderModule, FeatureContextBuilderModule, FeatureSwitchesModule, KafkaProducerModule, LoggerFactoryModule, MHMtlsParamsModule, ProductMixerFlagModule, ScalaObjectMapperModule, ThriftClientIdModule ) private val thriftClientModules = Seq( AnnQueryServiceClientModule, EarlybirdSearchClientModule, FrsClientModule, HydraPartitionClientModule, HydraRootClientModule, QigServiceClientModule, SimClustersAnnServiceClientModule, TweetyPieClientModule, UserAdGraphClientModule, UserTweetEntityGraphClientModule, UserTweetGraphClientModule, UserTweetGraphPlusClientModule, UserVideoGraphClientModule, ) private val grpcClientModules = Seq( NaviGRPCClientModule ) // Modules sorted alphabetically, please keep the order when adding a new module override val modules: Seq[Module] = coreModules ++ thriftClientModules ++ grpcClientModules ++ Seq( ActivePromotedTweetStoreModule, CertoStratoStoreModule, CertoTopicTweetSimilarityEngineModule, ConsumersBasedUserAdGraphSimilarityEngineModule, ConsumersBasedUserTweetGraphStoreModule, ConsumersBasedUserVideoGraphSimilarityEngineModule, ConsumersBasedUserVideoGraphStoreModule, ConsumerEmbeddingBasedTripSimilarityEngineModule, ConsumerEmbeddingBasedTwHINSimilarityEngineModule, ConsumerEmbeddingBasedTwoTowerSimilarityEngineModule, ConsumersBasedUserAdGraphStoreModule, ConsumerBasedWalsSimilarityEngineModule, DiffusionStoreModule, EmbeddingStoreModule, EarlybirdSimilarityEngineModule, EarlybirdRecencyBasedCandidateStoreModule, FrsStoreModule, HydraConfigModule, OfflineCandidateStoreModule, ProducerBasedUnifiedSimilarityEngineModule, ProducerBasedUserAdGraphSimilarityEngineModule, ProducerBasedUserTweetGraphSimilarityEngineModule, RealGraphOonStoreModule, RealGraphStoreMhModule, RepresentationManagerModule, RepresentationScorerModule, SimClustersANNServiceNameToClientMapper, SimClustersANNSimilarityEngineModule, SkitStratoStoreModule, SkitTopicTweetSimilarityEngineModule, StratoClientModule, StrongTiePredictionStoreModule, TimeoutConfigModule, TripCandidateStoreModule, TwiceClustersMembersStoreModule, TweetBasedQigSimilarityEngineModule, TweetBasedTwHINSimlarityEngineModule, TweetBasedUnifiedSimilarityEngineModule, TweetBasedUserAdGraphSimilarityEngineModule, TweetBasedUserTweetGraphSimilarityEngineModule, TweetBasedUserVideoGraphSimilarityEngineModule, TweetInfoStoreModule, TweetRecentEngagedUserStoreModule, TweetRecommendationResultsStoreModule, TwhinCollabFilterStratoStoreModule, TwhinCollabFilterLookupSimilarityEngineModule, UserSignalServiceColumnModule, UserSignalServiceStoreModule, UserStateStoreModule, UserTweetEntityGraphSimilarityEngineModule, DiffusionBasedSimilarityEngineModule, BlueVerifiedAnnotationStoreModule, new MtlsThriftWebFormsModule[st.CrMixer.MethodPerEndpoint](this) { override protected def defaultMethodAccess: MethodOptions.Access = { MethodOptions.Access.ByLdapGroup( Seq( "cr-mixer-admins", "recosplat-sensitive-data-medium", "recos-platform-admins", )) } } ) def configureThrift(router: ThriftRouter): Unit = { router .filter[LoggingMDCFilter] .filter[TraceIdMDCFilter] .filter[ThriftMDCFilter] .filter[ClientStatsFilter] .filter[AccessLoggingFilter] .filter[SetImpressedBucketsLocalContextFilter] .filter[ExceptionMappingFilter] .filter[Filter.TypeAgnostic, DarkTrafficFilterType] .exceptionMapper[LoggingThrowableExceptionMapper] .add[CrMixerThriftController] } override protected def warmup(): Unit = { handle[CrMixerThriftServerWarmupHandler]() handle[CrMixerHttpServerWarmupHandler]() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/CrMixerThriftServerWarmupHandler.scala ================================================ package com.twitter.cr_mixer import com.twitter.finagle.thrift.ClientId import com.twitter.finatra.thrift.routing.ThriftWarmup import com.twitter.inject.Logging import com.twitter.inject.utils.Handler import com.twitter.product_mixer.core.{thriftscala => pt} import com.twitter.cr_mixer.{thriftscala => st} import com.twitter.scrooge.Request import com.twitter.scrooge.Response import com.twitter.util.Return import com.twitter.util.Throw import com.twitter.util.Try import javax.inject.Inject import javax.inject.Singleton @Singleton class CrMixerThriftServerWarmupHandler @Inject() (warmup: ThriftWarmup) extends Handler with Logging { private val clientId = ClientId("thrift-warmup-client") def handle(): Unit = { val testIds = Seq(1, 2, 3) try { clientId.asCurrent { testIds.foreach { id => val warmupReq = warmupQuery(id) info(s"Sending warm-up request to service with query: $warmupReq") warmup.sendRequest( method = st.CrMixer.GetTweetRecommendations, req = Request(st.CrMixer.GetTweetRecommendations.Args(warmupReq)))(assertWarmupResponse) } } } catch { case e: Throwable => // we don't want a warmup failure to prevent start-up error(e.getMessage, e) } info("Warm-up done.") } private def warmupQuery(userId: Long): st.CrMixerTweetRequest = { val clientContext = pt.ClientContext( userId = Some(userId), guestId = None, appId = Some(258901L), ipAddress = Some("0.0.0.0"), userAgent = Some("FAKE_USER_AGENT_FOR_WARMUPS"), countryCode = Some("US"), languageCode = Some("en"), isTwoffice = None, userRoles = None, deviceId = Some("FAKE_DEVICE_ID_FOR_WARMUPS") ) st.CrMixerTweetRequest( clientContext = clientContext, product = st.Product.Home, productContext = Some(st.ProductContext.HomeContext(st.HomeContext())), ) } private def assertWarmupResponse( result: Try[Response[st.CrMixer.GetTweetRecommendations.SuccessType]] ): Unit = { // we collect and log any exceptions from the result. result match { case Return(_) => // ok case Throw(exception) => warn("Error performing warm-up request.") error(exception.getMessage, exception) } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/blender/AdsBlender.scala ================================================ package com.twitter.cr_mixer.blender import com.twitter.cr_mixer.model.BlendedAdsCandidate import com.twitter.cr_mixer.model.CandidateGenerationInfo import com.twitter.cr_mixer.model.InitialAdsCandidate import com.twitter.cr_mixer.util.InterleaveUtil import com.twitter.finagle.stats.StatsReceiver import com.twitter.simclusters_v2.common.TweetId import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton import scala.collection.mutable @Singleton case class AdsBlender @Inject() (globalStats: StatsReceiver) { private val name: String = this.getClass.getCanonicalName private val stats: StatsReceiver = globalStats.scope(name) /** * Interleaves candidates by iteratively choosing InterestedIn candidates and TWISTLY candidates * in turn. InterestedIn candidates have no source signal, whereas TWISTLY candidates do. TWISTLY * candidates themselves are interleaved by source before equal blending with InterestedIn * candidates. */ def blend( inputCandidates: Seq[Seq[InitialAdsCandidate]], ): Future[Seq[BlendedAdsCandidate]] = { // Filter out empty candidate sequence val candidates = inputCandidates.filter(_.nonEmpty) val (interestedInCandidates, twistlyCandidates) = candidates.partition(_.head.candidateGenerationInfo.sourceInfoOpt.isEmpty) // First interleave twistly candidates val interleavedTwistlyCandidates = InterleaveUtil.interleave(twistlyCandidates) val twistlyAndInterestedInCandidates = Seq(interestedInCandidates.flatten, interleavedTwistlyCandidates) // then interleave twistly candidates with interested in to make them even val interleavedCandidates = InterleaveUtil.interleave(twistlyAndInterestedInCandidates) stats.stat("candidates").add(interleavedCandidates.size) val blendedCandidates = buildBlendedAdsCandidate(inputCandidates, interleavedCandidates) Future.value(blendedCandidates) } private def buildBlendedAdsCandidate( inputCandidates: Seq[Seq[InitialAdsCandidate]], interleavedCandidates: Seq[InitialAdsCandidate] ): Seq[BlendedAdsCandidate] = { val cgInfoLookupMap = buildCandidateToCGInfosMap(inputCandidates) interleavedCandidates.map { interleavedCandidate => interleavedCandidate.toBlendedAdsCandidate(cgInfoLookupMap(interleavedCandidate.tweetId)) } } private def buildCandidateToCGInfosMap( candidateSeq: Seq[Seq[InitialAdsCandidate]], ): Map[TweetId, Seq[CandidateGenerationInfo]] = { val tweetIdMap = mutable.HashMap[TweetId, Seq[CandidateGenerationInfo]]() candidateSeq.foreach { candidates => candidates.foreach { candidate => val candidateGenerationInfoSeq = { tweetIdMap.getOrElse(candidate.tweetId, Seq.empty) } val candidateGenerationInfo = candidate.candidateGenerationInfo tweetIdMap.put( candidate.tweetId, candidateGenerationInfoSeq ++ Seq(candidateGenerationInfo)) } } tweetIdMap.toMap } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/blender/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/storehaus:core", "3rdparty/jvm/javax/inject:javax.inject", "3rdparty/src/jvm/com/twitter/storehaus:core", "configapi/configapi-core", "content-recommender/thrift/src/main/thrift:thrift-scala", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util", "cr-mixer/thrift/src/main/thrift:thrift-scala", "snowflake/src/main/scala/com/twitter/snowflake/id", "src/scala/com/twitter/simclusters_v2/common", "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/blender/BlendedCandidatesBuilder.scala ================================================ package com.twitter.cr_mixer.blender import com.twitter.cr_mixer.model.BlendedCandidate import com.twitter.cr_mixer.model.CandidateGenerationInfo import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.simclusters_v2.common.TweetId import scala.collection.mutable object BlendedCandidatesBuilder { /** * @param inputCandidates input candidate prior to interleaving * @param interleavedCandidates after interleaving. These tweets are de-duplicated. */ def build( inputCandidates: Seq[Seq[InitialCandidate]], interleavedCandidates: Seq[InitialCandidate] ): Seq[BlendedCandidate] = { val cgInfoLookupMap = buildCandidateToCGInfosMap(inputCandidates) interleavedCandidates.map { interleavedCandidate => interleavedCandidate.toBlendedCandidate(cgInfoLookupMap(interleavedCandidate.tweetId)) } } /** * The same tweet can be generated by different sources. * This function tells you which CandidateGenerationInfo generated a given tweet */ private def buildCandidateToCGInfosMap( candidateSeq: Seq[Seq[InitialCandidate]], ): Map[TweetId, Seq[CandidateGenerationInfo]] = { val tweetIdMap = mutable.HashMap[TweetId, Seq[CandidateGenerationInfo]]() candidateSeq.foreach { candidates => candidates.foreach { candidate => val candidateGenerationInfoSeq = { tweetIdMap.getOrElse(candidate.tweetId, Seq.empty) } val candidateGenerationInfo = candidate.candidateGenerationInfo tweetIdMap.put( candidate.tweetId, candidateGenerationInfoSeq ++ Seq(candidateGenerationInfo)) } } tweetIdMap.toMap } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/blender/ContentSignalBlender.scala ================================================ package com.twitter.cr_mixer.blender import com.twitter.cr_mixer.model.BlendedCandidate import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.param.BlenderParams import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.snowflake.id.SnowflakeId import com.twitter.timelines.configapi.Params import com.twitter.util.Future import com.twitter.util.Time import javax.inject.Inject case class ContentSignalBlender @Inject() (globalStats: StatsReceiver) { private val name: String = this.getClass.getCanonicalName private val stats: StatsReceiver = globalStats.scope(name) /** * Exposes multiple types of sorting relying only on Content Based signals * Candidate Recency, Random, FavoriteCount and finally Standardized, which standardizes the scores * that come from the active SimilarityEngine and then sort on the standardized scores. */ def blend( params: Params, inputCandidates: Seq[Seq[InitialCandidate]], ): Future[Seq[BlendedCandidate]] = { // Filter out empty candidate sequence val candidates = inputCandidates.filter(_.nonEmpty) val sortedCandidates = params(BlenderParams.ContentBlenderTypeSortingAlgorithmParam) match { case BlenderParams.ContentBasedSortingAlgorithmEnum.CandidateRecency => candidates.flatten.sortBy(c => getSnowflakeTimeStamp(c.tweetId)).reverse case BlenderParams.ContentBasedSortingAlgorithmEnum.RandomSorting => candidates.flatten.sortBy(_ => scala.util.Random.nextDouble()) case BlenderParams.ContentBasedSortingAlgorithmEnum.FavoriteCount => candidates.flatten.sortBy(-_.tweetInfo.favCount) case BlenderParams.ContentBasedSortingAlgorithmEnum.SimilarityToSignalSorting => standardizeAndSortByScore(flattenAndGroupByEngineTypeOrFirstContribEngine(candidates)) case _ => candidates.flatten.sortBy(-_.tweetInfo.favCount) } stats.stat("candidates").add(sortedCandidates.size) val blendedCandidates = BlendedCandidatesBuilder.build(inputCandidates, removeDuplicates(sortedCandidates)) Future.value(blendedCandidates) } private def removeDuplicates(candidates: Seq[InitialCandidate]): Seq[InitialCandidate] = { val seen = collection.mutable.Set.empty[Long] candidates.filter { c => if (seen.contains(c.tweetId)) { false } else { seen += c.tweetId true } } } private def groupByEngineTypeOrFirstContribEngine( candidates: Seq[InitialCandidate] ): Map[SimilarityEngineType, Seq[InitialCandidate]] = { val grouped = candidates.groupBy { candidate => val contrib = candidate.candidateGenerationInfo.contributingSimilarityEngines if (contrib.nonEmpty) { contrib.head.similarityEngineType } else { candidate.candidateGenerationInfo.similarityEngineInfo.similarityEngineType } } grouped } private def flattenAndGroupByEngineTypeOrFirstContribEngine( candidates: Seq[Seq[InitialCandidate]] ): Seq[Seq[InitialCandidate]] = { val flat = candidates.flatten val grouped = groupByEngineTypeOrFirstContribEngine(flat) grouped.values.toSeq } private def standardizeAndSortByScore( candidates: Seq[Seq[InitialCandidate]] ): Seq[InitialCandidate] = { candidates .map { innerSeq => val meanScore = innerSeq .map(c => c.candidateGenerationInfo.similarityEngineInfo.score.getOrElse(0.0)) .sum / innerSeq.length val stdDev = scala.math .sqrt( innerSeq .map(c => c.candidateGenerationInfo.similarityEngineInfo.score.getOrElse(0.0)) .map(a => a - meanScore) .map(a => a * a) .sum / innerSeq.length) innerSeq .map(c => ( c, c.candidateGenerationInfo.similarityEngineInfo.score .map { score => if (stdDev != 0) (score - meanScore) / stdDev else 0.0 } .getOrElse(0.0))) }.flatten.sortBy { case (_, standardizedScore) => -standardizedScore } .map { case (candidate, _) => candidate } } private def getSnowflakeTimeStamp(tweetId: Long): Time = { val isSnowflake = SnowflakeId.isSnowflakeId(tweetId) if (isSnowflake) { SnowflakeId(tweetId).time } else { Time.fromMilliseconds(0L) } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/blender/CountWeightedInterleaveBlender.scala ================================================ package com.twitter.cr_mixer.blender import com.twitter.cr_mixer.model.BlendedCandidate import com.twitter.cr_mixer.model.CrCandidateGeneratorQuery import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.param.BlenderParams import com.twitter.cr_mixer.util.CountWeightedInterleaveUtil import com.twitter.cr_mixer.util.InterleaveUtil import com.twitter.finagle.stats.StatsReceiver import com.twitter.timelines.configapi.Params import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton /** * A weighted round robin interleaving algorithm. * The weight of each blending group based on the count of candidates in each blending group. * The more candidates under a blending group, the more candidates are selected from it during round * robin, which in effect prioritizes this group. * * Weights sum up to 1. For example: * total candidates = 8 * Group Weight * [A1, A2, A3, A4] 4/8 = 0.5 // select 50% of results from group A * [B1, B2] 2/8 = 0.25 // 25% from group B * [C1, C2] 2/8 = 0.25 // 25% from group C * * Blended results = [A1, A2, B1, C1, A3, A4, B2, C2] * See @linht's go/weighted-interleave */ @Singleton case class CountWeightedInterleaveBlender @Inject() (globalStats: StatsReceiver) { import CountWeightedInterleaveBlender._ private val name: String = this.getClass.getCanonicalName private val stats: StatsReceiver = globalStats.scope(name) def blend( query: CrCandidateGeneratorQuery, inputCandidates: Seq[Seq[InitialCandidate]] ): Future[Seq[BlendedCandidate]] = { val weightedBlenderQuery = CountWeightedInterleaveBlender.paramToQuery(query.params) countWeightedInterleave(weightedBlenderQuery, inputCandidates) } private[blender] def countWeightedInterleave( query: WeightedBlenderQuery, inputCandidates: Seq[Seq[InitialCandidate]], ): Future[Seq[BlendedCandidate]] = { val candidatesAndWeightKeyByIndexId: Seq[(Seq[InitialCandidate], Double)] = { CountWeightedInterleaveUtil.buildInitialCandidatesWithWeightKeyByFeature( inputCandidates, query.rankerWeightShrinkage) } val interleavedCandidates = InterleaveUtil.weightedInterleave(candidatesAndWeightKeyByIndexId, query.maxWeightAdjustments) stats.stat("candidates").add(interleavedCandidates.size) val blendedCandidates = BlendedCandidatesBuilder.build(inputCandidates, interleavedCandidates) Future.value(blendedCandidates) } } object CountWeightedInterleaveBlender { /** * We pass two parameters to the weighted interleaver: * @param rankerWeightShrinkage shrinkage parameter between [0, 1] that determines how close we * stay to uniform sampling. The bigger the shrinkage the * closer we are to uniform round robin * @param maxWeightAdjustments max number of weighted sampling to do prior to defaulting to * uniform. Set so that we avoid infinite loops (e.g. if weights are * 0) */ case class WeightedBlenderQuery( rankerWeightShrinkage: Double, maxWeightAdjustments: Int) def paramToQuery(params: Params): WeightedBlenderQuery = { val rankerWeightShrinkage: Double = params(BlenderParams.RankingInterleaveWeightShrinkageParam) val maxWeightAdjustments: Int = params(BlenderParams.RankingInterleaveMaxWeightAdjustments) WeightedBlenderQuery(rankerWeightShrinkage, maxWeightAdjustments) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/blender/InterleaveBlender.scala ================================================ package com.twitter.cr_mixer.blender import com.twitter.cr_mixer.model.BlendedCandidate import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.util.InterleaveUtil import com.twitter.finagle.stats.StatsReceiver import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton @Singleton case class InterleaveBlender @Inject() (globalStats: StatsReceiver) { private val name: String = this.getClass.getCanonicalName private val stats: StatsReceiver = globalStats.scope(name) /** * Interleaves candidates, by taking 1 candidate from each Seq[Seq[InitialCandidate]] in sequence, * until we run out of candidates. */ def blend( inputCandidates: Seq[Seq[InitialCandidate]], ): Future[Seq[BlendedCandidate]] = { val interleavedCandidates = InterleaveUtil.interleave(inputCandidates) stats.stat("candidates").add(interleavedCandidates.size) val blendedCandidates = BlendedCandidatesBuilder.build(inputCandidates, interleavedCandidates) Future.value(blendedCandidates) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/blender/SourceTypeBackFillBlender.scala ================================================ package com.twitter.cr_mixer.blender import com.twitter.cr_mixer.blender.ImplicitSignalBackFillBlender.BackFillSourceTypes import com.twitter.cr_mixer.blender.ImplicitSignalBackFillBlender.BackFillSourceTypesWithVideo import com.twitter.cr_mixer.model.BlendedCandidate import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.param.BlenderParams import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.cr_mixer.util.InterleaveUtil import com.twitter.finagle.stats.StatsReceiver import com.twitter.timelines.configapi.Params import com.twitter.util.Future import javax.inject.Inject case class SourceTypeBackFillBlender @Inject() (globalStats: StatsReceiver) { private val name: String = this.getClass.getCanonicalName private val stats: StatsReceiver = globalStats.scope(name) /** * Partition the candidates based on source type * Interleave the two partitions of candidates separately * Then append the back fill candidates to the end */ def blend( params: Params, inputCandidates: Seq[Seq[InitialCandidate]], ): Future[Seq[BlendedCandidate]] = { // Filter out empty candidate sequence val candidates = inputCandidates.filter(_.nonEmpty) val backFillSourceTypes = if (params(BlenderParams.SourceTypeBackFillEnableVideoBackFill)) BackFillSourceTypesWithVideo else BackFillSourceTypes // partition candidates based on their source types val (backFillCandidates, regularCandidates) = candidates.partition( _.head.candidateGenerationInfo.sourceInfoOpt .exists(sourceInfo => backFillSourceTypes.contains(sourceInfo.sourceType))) val interleavedRegularCandidates = InterleaveUtil.interleave(regularCandidates) val interleavedBackFillCandidates = InterleaveUtil.interleave(backFillCandidates) stats.stat("backFillCandidates").add(interleavedBackFillCandidates.size) // Append interleaved backfill candidates to the end val interleavedCandidates = interleavedRegularCandidates ++ interleavedBackFillCandidates stats.stat("candidates").add(interleavedCandidates.size) val blendedCandidates = BlendedCandidatesBuilder.build(inputCandidates, interleavedCandidates) Future.value(blendedCandidates) } } object ImplicitSignalBackFillBlender { final val BackFillSourceTypesWithVideo: Set[SourceType] = Set( SourceType.UserRepeatedProfileVisit, SourceType.VideoTweetPlayback50, SourceType.VideoTweetQualityView) final val BackFillSourceTypes: Set[SourceType] = Set(SourceType.UserRepeatedProfileVisit) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/blender/SwitchBlender.scala ================================================ package com.twitter.cr_mixer.blender import com.twitter.core_workflows.user_model.thriftscala.UserState import com.twitter.cr_mixer.model.BlendedCandidate import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.param.BlenderParams import com.twitter.cr_mixer.param.BlenderParams.BlendingAlgorithmEnum import com.twitter.finagle.stats.StatsReceiver import com.twitter.timelines.configapi.Params import com.twitter.util.Future import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton @Singleton case class SwitchBlender @Inject() ( defaultBlender: InterleaveBlender, sourceTypeBackFillBlender: SourceTypeBackFillBlender, adsBlender: AdsBlender, contentSignalBlender: ContentSignalBlender, globalStats: StatsReceiver) { private val stats = globalStats.scope(this.getClass.getCanonicalName) def blend( params: Params, userState: UserState, inputCandidates: Seq[Seq[InitialCandidate]], ): Future[Seq[BlendedCandidate]] = { // Take out empty seq val nonEmptyCandidates = inputCandidates.collect { case candidates if candidates.nonEmpty => candidates } stats.stat("num_of_sequences").add(inputCandidates.size) // Sort the seqs in an order val innerSignalSorting = params(BlenderParams.SignalTypeSortingAlgorithmParam) match { case BlenderParams.ContentBasedSortingAlgorithmEnum.SourceSignalRecency => SwitchBlender.TimestampOrder case BlenderParams.ContentBasedSortingAlgorithmEnum.RandomSorting => SwitchBlender.RandomOrder case _ => SwitchBlender.TimestampOrder } val candidatesToBlend = nonEmptyCandidates.sortBy(_.head)(innerSignalSorting) // Blend based on specified blender rules params(BlenderParams.BlendingAlgorithmParam) match { case BlendingAlgorithmEnum.RoundRobin => defaultBlender.blend(candidatesToBlend) case BlendingAlgorithmEnum.SourceTypeBackFill => sourceTypeBackFillBlender.blend(params, candidatesToBlend) case BlendingAlgorithmEnum.SourceSignalSorting => contentSignalBlender.blend(params, candidatesToBlend) case _ => defaultBlender.blend(candidatesToBlend) } } } object SwitchBlender { /** * Prefers candidates generated from sources with the latest timestamps. * The newer the source signal, the higher a candidate ranks. * This ordering biases against consumer-based candidates because their timestamp defaults to 0 * * Within a Seq[Seq[Candidate]], all candidates within a inner Seq * are guaranteed to have the same sourceInfo because they are grouped by (sourceInfo, SE model). * Hence, we can pick .headOption to represent the whole list when filtering by the internalId of the sourceInfoOpt. * But of course the similarityEngine score in a CGInfo could be different. */ val TimestampOrder: Ordering[InitialCandidate] = math.Ordering .by[InitialCandidate, Time]( _.candidateGenerationInfo.sourceInfoOpt .flatMap(_.sourceEventTime) .getOrElse(Time.fromMilliseconds(0L))) .reverse private val RandomOrder: Ordering[InitialCandidate] = Ordering.by[InitialCandidate, Double](_ => scala.util.Random.nextDouble()) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation/AdsCandidateGenerator.scala ================================================ package com.twitter.cr_mixer.candidate_generation import com.twitter.cr_mixer.blender.AdsBlender import com.twitter.cr_mixer.logging.AdsRecommendationsScribeLogger import com.twitter.cr_mixer.model.AdsCandidateGeneratorQuery import com.twitter.cr_mixer.model.BlendedAdsCandidate import com.twitter.cr_mixer.model.InitialAdsCandidate import com.twitter.cr_mixer.model.RankedAdsCandidate import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.param.AdsParams import com.twitter.cr_mixer.param.ConsumersBasedUserAdGraphParams import com.twitter.cr_mixer.source_signal.RealGraphInSourceGraphFetcher import com.twitter.cr_mixer.source_signal.SourceFetcher.FetcherQuery import com.twitter.cr_mixer.source_signal.UssSourceSignalFetcher import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.simclusters_v2.common.UserId import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton @Singleton class AdsCandidateGenerator @Inject() ( ussSourceSignalFetcher: UssSourceSignalFetcher, realGraphInSourceGraphFetcher: RealGraphInSourceGraphFetcher, adsCandidateSourceRouter: AdsCandidateSourcesRouter, adsBlender: AdsBlender, scribeLogger: AdsRecommendationsScribeLogger, globalStats: StatsReceiver) { private val stats: StatsReceiver = globalStats.scope(this.getClass.getCanonicalName) private val fetchSourcesStats = stats.scope("fetchSources") private val fetchRealGraphSeedsStats = stats.scope("fetchRealGraphSeeds") private val fetchCandidatesStats = stats.scope("fetchCandidates") private val interleaveStats = stats.scope("interleave") private val rankStats = stats.scope("rank") def get(query: AdsCandidateGeneratorQuery): Future[Seq[RankedAdsCandidate]] = { val allStats = stats.scope("all") val perProductStats = stats.scope("perProduct", query.product.toString) StatsUtil.trackItemsStats(allStats) { StatsUtil.trackItemsStats(perProductStats) { for { // fetch source signals sourceSignals <- StatsUtil.trackBlockStats(fetchSourcesStats) { fetchSources(query) } realGraphSeeds <- StatsUtil.trackItemMapStats(fetchRealGraphSeedsStats) { fetchSeeds(query) } // get initial candidates from similarity engines // hydrate lineItemInfo and filter out non active ads initialCandidates <- StatsUtil.trackBlockStats(fetchCandidatesStats) { fetchCandidates(query, sourceSignals, realGraphSeeds) } // blend candidates blendedCandidates <- StatsUtil.trackItemsStats(interleaveStats) { interleave(initialCandidates) } rankedCandidates <- StatsUtil.trackItemsStats(rankStats) { rank( blendedCandidates, query.params(AdsParams.EnableScoreBoost), query.params(AdsParams.AdsCandidateGenerationScoreBoostFactor), rankStats) } } yield { rankedCandidates.take(query.maxNumResults) } } } } def fetchSources( query: AdsCandidateGeneratorQuery ): Future[Set[SourceInfo]] = { val fetcherQuery = FetcherQuery(query.userId, query.product, query.userState, query.params) ussSourceSignalFetcher.get(fetcherQuery).map(_.getOrElse(Seq.empty).toSet) } private def fetchCandidates( query: AdsCandidateGeneratorQuery, sourceSignals: Set[SourceInfo], realGraphSeeds: Map[UserId, Double] ): Future[Seq[Seq[InitialAdsCandidate]]] = { scribeLogger.scribeInitialAdsCandidates( query, adsCandidateSourceRouter .fetchCandidates(query.userId, sourceSignals, realGraphSeeds, query.params), query.params(AdsParams.EnableScribe) ) } private def fetchSeeds( query: AdsCandidateGeneratorQuery ): Future[Map[UserId, Double]] = { if (query.params(ConsumersBasedUserAdGraphParams.EnableSourceParam)) { realGraphInSourceGraphFetcher .get(FetcherQuery(query.userId, query.product, query.userState, query.params)) .map(_.map(_.seedWithScores).getOrElse(Map.empty)) } else Future.value(Map.empty[UserId, Double]) } private def interleave( candidates: Seq[Seq[InitialAdsCandidate]] ): Future[Seq[BlendedAdsCandidate]] = { adsBlender .blend(candidates) } private def rank( candidates: Seq[BlendedAdsCandidate], enableScoreBoost: Boolean, scoreBoostFactor: Double, statsReceiver: StatsReceiver, ): Future[Seq[RankedAdsCandidate]] = { val candidateSize = candidates.size val rankedCandidates = candidates.zipWithIndex.map { case (candidate, index) => val score = 0.5 + 0.5 * ((candidateSize - index).toDouble / candidateSize) val boostedScore = if (enableScoreBoost) { statsReceiver.stat("boostedScore").add((100.0 * score * scoreBoostFactor).toFloat) score * scoreBoostFactor } else { statsReceiver.stat("score").add((100.0 * score).toFloat) score } candidate.toRankedAdsCandidate(boostedScore) } Future.value(rankedCandidates) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation/AdsCandidateSourcesRouter.scala ================================================ package com.twitter.cr_mixer.candidate_generation import com.twitter.cr_mixer.model.CandidateGenerationInfo import com.twitter.cr_mixer.model.InitialAdsCandidate import com.twitter.cr_mixer.model.ModelConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.model.TweetWithCandidateGenerationInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.ConsumersBasedUserAdGraphParams import com.twitter.cr_mixer.param.ConsumerBasedWalsParams import com.twitter.cr_mixer.param.ConsumerEmbeddingBasedCandidateGenerationParams import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.param.InterestedInParams import com.twitter.cr_mixer.param.ProducerBasedCandidateGenerationParams import com.twitter.cr_mixer.param.SimClustersANNParams import com.twitter.cr_mixer.param.TweetBasedCandidateGenerationParams import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.similarity_engine.ConsumerBasedWalsSimilarityEngine import com.twitter.cr_mixer.similarity_engine.ConsumersBasedUserAdGraphSimilarityEngine import com.twitter.cr_mixer.similarity_engine.FilterUtil import com.twitter.cr_mixer.similarity_engine.HnswANNEngineQuery import com.twitter.cr_mixer.similarity_engine.HnswANNSimilarityEngine import com.twitter.cr_mixer.similarity_engine.ProducerBasedUserAdGraphSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimClustersANNSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimClustersANNSimilarityEngine.Query import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.similarity_engine.TweetBasedUserAdGraphSimilarityEngine import com.twitter.cr_mixer.thriftscala.LineItemInfo import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.finagle.stats.StatsReceiver import com.twitter.simclusters_v2.common.ModelVersions import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.common.UserId import com.twitter.simclusters_v2.thriftscala.EmbeddingType import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.timelines.configapi.Params import com.twitter.util.Future import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton case class AdsCandidateSourcesRouter @Inject() ( activePromotedTweetStore: ReadableStore[TweetId, Seq[LineItemInfo]], decider: CrMixerDecider, @Named(ModuleNames.SimClustersANNSimilarityEngine) simClustersANNSimilarityEngine: StandardSimilarityEngine[ Query, TweetWithScore ], @Named(ModuleNames.TweetBasedUserAdGraphSimilarityEngine) tweetBasedUserAdGraphSimilarityEngine: StandardSimilarityEngine[ TweetBasedUserAdGraphSimilarityEngine.Query, TweetWithScore ], @Named(ModuleNames.ConsumersBasedUserAdGraphSimilarityEngine) consumersBasedUserAdGraphSimilarityEngine: StandardSimilarityEngine[ ConsumersBasedUserAdGraphSimilarityEngine.Query, TweetWithScore ], @Named(ModuleNames.ProducerBasedUserAdGraphSimilarityEngine) producerBasedUserAdGraphSimilarityEngine: StandardSimilarityEngine[ ProducerBasedUserAdGraphSimilarityEngine.Query, TweetWithScore ], @Named(ModuleNames.TweetBasedTwHINANNSimilarityEngine) tweetBasedTwHINANNSimilarityEngine: HnswANNSimilarityEngine, @Named(ModuleNames.ConsumerEmbeddingBasedTwHINANNSimilarityEngine) consumerTwHINANNSimilarityEngine: HnswANNSimilarityEngine, @Named(ModuleNames.ConsumerBasedWalsSimilarityEngine) consumerBasedWalsSimilarityEngine: StandardSimilarityEngine[ ConsumerBasedWalsSimilarityEngine.Query, TweetWithScore ], globalStats: StatsReceiver, ) { import AdsCandidateSourcesRouter._ val stats: StatsReceiver = globalStats.scope(this.getClass.getSimpleName) def fetchCandidates( requestUserId: UserId, sourceSignals: Set[SourceInfo], realGraphSeeds: Map[UserId, Double], params: configapi.Params ): Future[Seq[Seq[InitialAdsCandidate]]] = { val simClustersANN1ConfigId = params(SimClustersANNParams.SimClustersANN1ConfigId) val tweetBasedSANNMinScore = params( TweetBasedCandidateGenerationParams.SimClustersMinScoreParam) val tweetBasedSANN1Candidates = if (params(TweetBasedCandidateGenerationParams.EnableSimClustersANN1Param)) { Future.collect( CandidateSourcesRouter.getTweetBasedSourceInfo(sourceSignals).toSeq.map { sourceInfo => getSimClustersANNCandidates( requestUserId, Some(sourceInfo), params, simClustersANN1ConfigId, tweetBasedSANNMinScore) }) } else Future.value(Seq.empty) val simClustersANN2ConfigId = params(SimClustersANNParams.SimClustersANN2ConfigId) val tweetBasedSANN2Candidates = if (params(TweetBasedCandidateGenerationParams.EnableSimClustersANN2Param)) { Future.collect( CandidateSourcesRouter.getTweetBasedSourceInfo(sourceSignals).toSeq.map { sourceInfo => getSimClustersANNCandidates( requestUserId, Some(sourceInfo), params, simClustersANN2ConfigId, tweetBasedSANNMinScore) }) } else Future.value(Seq.empty) val tweetBasedUagCandidates = if (params(TweetBasedCandidateGenerationParams.EnableUAGParam)) { Future.collect( CandidateSourcesRouter.getTweetBasedSourceInfo(sourceSignals).toSeq.map { sourceInfo => getTweetBasedUserAdGraphCandidates(Some(sourceInfo), params) }) } else Future.value(Seq.empty) val realGraphInNetworkBasedUagCandidates = if (params(ConsumersBasedUserAdGraphParams.EnableSourceParam)) { getRealGraphConsumersBasedUserAdGraphCandidates(realGraphSeeds, params).map(Seq(_)) } else Future.value(Seq.empty) val producerBasedUagCandidates = if (params(ProducerBasedCandidateGenerationParams.EnableUAGParam)) { Future.collect( CandidateSourcesRouter.getProducerBasedSourceInfo(sourceSignals).toSeq.map { sourceInfo => getProducerBasedUserAdGraphCandidates(Some(sourceInfo), params) }) } else Future.value(Seq.empty) val tweetBasedTwhinAdsCandidates = if (params(TweetBasedCandidateGenerationParams.EnableTwHINParam)) { Future.collect( CandidateSourcesRouter.getTweetBasedSourceInfo(sourceSignals).toSeq.map { sourceInfo => getTwHINAdsCandidates( tweetBasedTwHINANNSimilarityEngine, SimilarityEngineType.TweetBasedTwHINANN, requestUserId, Some(sourceInfo), ModelConfig.DebuggerDemo) }) } else Future.value(Seq.empty) val producerBasedSANNMinScore = params( ProducerBasedCandidateGenerationParams.SimClustersMinScoreParam) val producerBasedSANN1Candidates = if (params(ProducerBasedCandidateGenerationParams.EnableSimClustersANN1Param)) { Future.collect( CandidateSourcesRouter.getProducerBasedSourceInfo(sourceSignals).toSeq.map { sourceInfo => getSimClustersANNCandidates( requestUserId, Some(sourceInfo), params, simClustersANN1ConfigId, producerBasedSANNMinScore) }) } else Future.value(Seq.empty) val producerBasedSANN2Candidates = if (params(ProducerBasedCandidateGenerationParams.EnableSimClustersANN2Param)) { Future.collect( CandidateSourcesRouter.getProducerBasedSourceInfo(sourceSignals).toSeq.map { sourceInfo => getSimClustersANNCandidates( requestUserId, Some(sourceInfo), params, simClustersANN2ConfigId, producerBasedSANNMinScore) }) } else Future.value(Seq.empty) val interestedInMinScore = params(InterestedInParams.MinScoreParam) val interestedInSANN1Candidates = if (params(InterestedInParams.EnableSimClustersANN1Param)) { getSimClustersANNCandidates( requestUserId, None, params, simClustersANN1ConfigId, interestedInMinScore).map(Seq(_)) } else Future.value(Seq.empty) val interestedInSANN2Candidates = if (params(InterestedInParams.EnableSimClustersANN2Param)) { getSimClustersANNCandidates( requestUserId, None, params, simClustersANN2ConfigId, interestedInMinScore).map(Seq(_)) } else Future.value(Seq.empty) val consumerTwHINAdsCandidates = if (params(ConsumerEmbeddingBasedCandidateGenerationParams.EnableTwHINParam)) { getTwHINAdsCandidates( consumerTwHINANNSimilarityEngine, SimilarityEngineType.ConsumerEmbeddingBasedTwHINANN, requestUserId, None, ModelConfig.DebuggerDemo).map(Seq(_)) } else Future.value(Seq.empty) val consumerBasedWalsCandidates = if (params( ConsumerBasedWalsParams.EnableSourceParam )) { getConsumerBasedWalsCandidates(sourceSignals, params) }.map { Seq(_) } else Future.value(Seq.empty) Future .collect(Seq( tweetBasedSANN1Candidates, tweetBasedSANN2Candidates, tweetBasedUagCandidates, tweetBasedTwhinAdsCandidates, producerBasedUagCandidates, producerBasedSANN1Candidates, producerBasedSANN2Candidates, realGraphInNetworkBasedUagCandidates, interestedInSANN1Candidates, interestedInSANN2Candidates, consumerTwHINAdsCandidates, consumerBasedWalsCandidates, )).map(_.flatten).map { tweetsWithCGInfoSeq => Future.collect( tweetsWithCGInfoSeq.map(candidates => convertToInitialCandidates(candidates, stats))) }.flatten.map { candidatesLists => val result = candidatesLists.filter(_.nonEmpty) stats.stat("numOfSequences").add(result.size) stats.stat("flattenCandidatesWithDup").add(result.flatten.size) result } } private[candidate_generation] def convertToInitialCandidates( candidates: Seq[TweetWithCandidateGenerationInfo], stats: StatsReceiver ): Future[Seq[InitialAdsCandidate]] = { val tweetIds = candidates.map(_.tweetId).toSet stats.stat("initialCandidateSizeBeforeLineItemFilter").add(tweetIds.size) Future.collect(activePromotedTweetStore.multiGet(tweetIds)).map { lineItemInfos => /** * * If lineItemInfo does not exist, we will filter out the promoted tweet as it cannot be targeted and ranked in admixer */ val filteredCandidates = candidates.collect { case candidate if lineItemInfos.getOrElse(candidate.tweetId, None).isDefined => val lineItemInfo = lineItemInfos(candidate.tweetId) .getOrElse(throw new IllegalStateException("Check previous line's condition")) InitialAdsCandidate( tweetId = candidate.tweetId, lineItemInfo = lineItemInfo, candidate.candidateGenerationInfo ) } stats.stat("initialCandidateSizeAfterLineItemFilter").add(filteredCandidates.size) filteredCandidates } } private[candidate_generation] def getSimClustersANNCandidates( requestUserId: UserId, sourceInfo: Option[SourceInfo], params: configapi.Params, configId: String, minScore: Double ) = { val simClustersModelVersion = ModelVersions.Enum.enumToSimClustersModelVersionMap(params(GlobalParams.ModelVersionParam)) val embeddingType = if (sourceInfo.isEmpty) { params(InterestedInParams.InterestedInEmbeddingIdParam).embeddingType } else getSimClustersANNEmbeddingType(sourceInfo.get) val query = SimClustersANNSimilarityEngine.fromParams( if (sourceInfo.isEmpty) InternalId.UserId(requestUserId) else sourceInfo.get.internalId, embeddingType, simClustersModelVersion, configId, params ) // dark traffic to simclusters-ann-2 if (decider.isAvailable(DeciderConstants.enableSimClustersANN2DarkTrafficDeciderKey)) { val simClustersANN2ConfigId = params(SimClustersANNParams.SimClustersANN2ConfigId) val sann2Query = SimClustersANNSimilarityEngine.fromParams( if (sourceInfo.isEmpty) InternalId.UserId(requestUserId) else sourceInfo.get.internalId, embeddingType, simClustersModelVersion, simClustersANN2ConfigId, params ) simClustersANNSimilarityEngine .getCandidates(sann2Query) } simClustersANNSimilarityEngine .getCandidates(query).map(_.getOrElse(Seq.empty)).map(_.filter(_.score > minScore).map { tweetWithScore => val similarityEngineInfo = SimClustersANNSimilarityEngine .toSimilarityEngineInfo(query, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( sourceInfo, similarityEngineInfo, Seq(similarityEngineInfo) )) }) } private[candidate_generation] def getProducerBasedUserAdGraphCandidates( sourceInfo: Option[SourceInfo], params: configapi.Params ) = { val query = ProducerBasedUserAdGraphSimilarityEngine.fromParams( sourceInfo.get.internalId, params ) producerBasedUserAdGraphSimilarityEngine .getCandidates(query).map(_.getOrElse(Seq.empty)).map(_.map { tweetWithScore => val similarityEngineInfo = ProducerBasedUserAdGraphSimilarityEngine .toSimilarityEngineInfo(tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( sourceInfo, similarityEngineInfo, Seq(similarityEngineInfo) )) }) } private[candidate_generation] def getTweetBasedUserAdGraphCandidates( sourceInfo: Option[SourceInfo], params: configapi.Params ) = { val query = TweetBasedUserAdGraphSimilarityEngine.fromParams( sourceInfo.get.internalId, params ) tweetBasedUserAdGraphSimilarityEngine .getCandidates(query).map(_.getOrElse(Seq.empty)).map(_.map { tweetWithScore => val similarityEngineInfo = TweetBasedUserAdGraphSimilarityEngine .toSimilarityEngineInfo(tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( sourceInfo, similarityEngineInfo, Seq(similarityEngineInfo) )) }) } private[candidate_generation] def getRealGraphConsumersBasedUserAdGraphCandidates( realGraphSeeds: Map[UserId, Double], params: configapi.Params ) = { val query = ConsumersBasedUserAdGraphSimilarityEngine .fromParams(realGraphSeeds, params) // The internalId is a placeholder value. We do not plan to store the full seedUserId set. val sourceInfo = SourceInfo( sourceType = SourceType.RealGraphIn, internalId = InternalId.UserId(0L), sourceEventTime = None ) consumersBasedUserAdGraphSimilarityEngine .getCandidates(query).map(_.getOrElse(Seq.empty)).map(_.map { tweetWithScore => val similarityEngineInfo = ConsumersBasedUserAdGraphSimilarityEngine .toSimilarityEngineInfo(tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(sourceInfo), similarityEngineInfo, Seq.empty // Atomic Similarity Engine. Hence it has no contributing SEs ) ) }) } private[candidate_generation] def getTwHINAdsCandidates( similarityEngine: HnswANNSimilarityEngine, similarityEngineType: SimilarityEngineType, requestUserId: UserId, sourceInfo: Option[SourceInfo], // if none, then it's consumer-based similarity engine model: String ): Future[Seq[TweetWithCandidateGenerationInfo]] = { val internalId = if (sourceInfo.nonEmpty) sourceInfo.get.internalId else InternalId.UserId(requestUserId) similarityEngine .getCandidates(buildHnswANNQuery(internalId, model)).map(_.getOrElse(Seq.empty)).map(_.map { tweetWithScore => val similarityEngineInfo = SimilarityEngineInfo( similarityEngineType = similarityEngineType, modelId = Some(model), score = Some(tweetWithScore.score)) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( None, similarityEngineInfo, Seq(similarityEngineInfo) )) }) } private[candidate_generation] def getConsumerBasedWalsCandidates( sourceSignals: Set[SourceInfo], params: configapi.Params ): Future[Seq[TweetWithCandidateGenerationInfo]] = { // Fetch source signals and filter them based on age. val signals = FilterUtil.tweetSourceAgeFilter( getConsumerBasedWalsSourceInfo(sourceSignals).toSeq, params(ConsumerBasedWalsParams.MaxTweetSignalAgeHoursParam)) val candidatesOptFut = consumerBasedWalsSimilarityEngine.getCandidates( ConsumerBasedWalsSimilarityEngine.fromParams(signals, params) ) val tweetsWithCandidateGenerationInfoOptFut = candidatesOptFut.map { _.map { tweetsWithScores => val sortedCandidates = tweetsWithScores.sortBy(-_.score) val filteredCandidates = FilterUtil.tweetAgeFilter(sortedCandidates, params(GlobalParams.MaxTweetAgeHoursParam)) consumerBasedWalsSimilarityEngine.getScopedStats .stat("filteredCandidates_size").add(filteredCandidates.size) val tweetsWithCandidateGenerationInfo = filteredCandidates.map { tweetWithScore => { val similarityEngineInfo = ConsumerBasedWalsSimilarityEngine.toSimilarityEngineInfo(tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( None, similarityEngineInfo, Seq.empty // Atomic Similarity Engine. Hence it has no contributing SEs ) ) } } val maxCandidateNum = params(GlobalParams.MaxCandidateNumPerSourceKeyParam) tweetsWithCandidateGenerationInfo.take(maxCandidateNum) } } for { tweetsWithCandidateGenerationInfoOpt <- tweetsWithCandidateGenerationInfoOptFut } yield tweetsWithCandidateGenerationInfoOpt.toSeq.flatten } } object AdsCandidateSourcesRouter { def getSimClustersANNEmbeddingType( sourceInfo: SourceInfo ): EmbeddingType = { sourceInfo.sourceType match { case SourceType.TweetFavorite | SourceType.Retweet | SourceType.OriginalTweet | SourceType.Reply | SourceType.TweetShare | SourceType.NotificationClick | SourceType.GoodTweetClick | SourceType.VideoTweetQualityView | SourceType.VideoTweetPlayback50 => EmbeddingType.LogFavLongestL2EmbeddingTweet case SourceType.UserFollow | SourceType.UserRepeatedProfileVisit | SourceType.RealGraphOon | SourceType.FollowRecommendation | SourceType.UserTrafficAttributionProfileVisit | SourceType.GoodProfileClick | SourceType.TwiceUserId => EmbeddingType.FavBasedProducer case _ => throw new IllegalArgumentException("sourceInfo.sourceType not supported") } } def buildHnswANNQuery(internalId: InternalId, modelId: String): HnswANNEngineQuery = { HnswANNEngineQuery( sourceId = internalId, modelId = modelId, params = Params.Empty ) } def getConsumerBasedWalsSourceInfo( sourceSignals: Set[SourceInfo] ): Set[SourceInfo] = { val AllowedSourceTypesForConsumerBasedWalsSE = Set( SourceType.TweetFavorite.value, SourceType.Retweet.value, SourceType.TweetDontLike.value, //currently no-op SourceType.TweetReport.value, //currently no-op SourceType.AccountMute.value, //currently no-op SourceType.AccountBlock.value //currently no-op ) sourceSignals.collect { case sourceInfo if AllowedSourceTypesForConsumerBasedWalsSE.contains(sourceInfo.sourceType.value) => sourceInfo } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/storehaus:core", "3rdparty/jvm/javax/inject:javax.inject", "3rdparty/src/jvm/com/twitter/storehaus:core", "ann/src/main/scala/com/twitter/ann/hnsw", "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", "configapi/configapi-core", "content-recommender/thrift/src/main/thrift:thrift-scala", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/blender", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/config", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/filter", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util", "cr-mixer/thrift/src/main/thrift:thrift-scala", "cuad/projects/hashspace/thrift:thrift-scala", "decider/src/main/scala", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "frigate/frigate-common:base", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/base", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/candidate", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util:stats_util", "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", "hermit/hermit-core/src/main/scala/com/twitter/hermit/model", "simclusters-ann/thrift/src/main/thrift:thrift-scala", "snowflake/src/main/scala/com/twitter/snowflake/id", "src/scala/com/twitter/cortex/ml/embeddings/common:Helpers", "src/scala/com/twitter/ml/featurestore/lib", "src/scala/com/twitter/simclusters_v2/common", "src/thrift/com/twitter/frigate/data_pipeline/scalding:blue_verified_annotations-scala", "src/thrift/com/twitter/ml/api:embedding-scala", "src/thrift/com/twitter/recos/user_tweet_graph:user_tweet_graph-scala", "src/thrift/com/twitter/recos/user_tweet_graph_plus:user_tweet_graph_plus-scala", "src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/search/query_interaction_graph/service:qig-service-scala", "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", "strato/config/columns/cuad/hashspace:hashspace-strato-client", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation/CandidateSourcesRouter.scala ================================================ package com.twitter.cr_mixer.candidate_generation import com.twitter.contentrecommender.thriftscala.TweetInfo import com.twitter.cr_mixer.model.CandidateGenerationInfo import com.twitter.cr_mixer.model.GraphSourceInfo import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.model.ModelConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.model.TripTweetWithScore import com.twitter.cr_mixer.model.TweetWithCandidateGenerationInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.model.TweetWithScoreAndSocialProof import com.twitter.cr_mixer.param.ConsumerBasedWalsParams import com.twitter.cr_mixer.param.ConsumerEmbeddingBasedCandidateGenerationParams import com.twitter.cr_mixer.param.ConsumersBasedUserVideoGraphParams import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.similarity_engine.ConsumersBasedUserVideoGraphSimilarityEngine import com.twitter.cr_mixer.similarity_engine.ConsumerBasedWalsSimilarityEngine import com.twitter.cr_mixer.similarity_engine.ConsumerEmbeddingBasedTripSimilarityEngine import com.twitter.cr_mixer.similarity_engine.ConsumerEmbeddingBasedTwHINSimilarityEngine import com.twitter.cr_mixer.similarity_engine.ConsumerEmbeddingBasedTwoTowerSimilarityEngine import com.twitter.cr_mixer.similarity_engine.EngineQuery import com.twitter.cr_mixer.similarity_engine.FilterUtil import com.twitter.cr_mixer.similarity_engine.HnswANNEngineQuery import com.twitter.cr_mixer.similarity_engine.HnswANNSimilarityEngine import com.twitter.cr_mixer.similarity_engine.ProducerBasedUnifiedSimilarityEngine import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.similarity_engine.TripEngineQuery import com.twitter.cr_mixer.similarity_engine.TweetBasedUnifiedSimilarityEngine import com.twitter.cr_mixer.similarity_engine.UserTweetEntityGraphSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.finagle.stats.StatsReceiver import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.common.UserId import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.util.Future import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton /** * Route the SourceInfo to the associated Candidate Engines. */ @Singleton case class CandidateSourcesRouter @Inject() ( customizedRetrievalCandidateGeneration: CustomizedRetrievalCandidateGeneration, simClustersInterestedInCandidateGeneration: SimClustersInterestedInCandidateGeneration, @Named(ModuleNames.TweetBasedUnifiedSimilarityEngine) tweetBasedUnifiedSimilarityEngine: StandardSimilarityEngine[ TweetBasedUnifiedSimilarityEngine.Query, TweetWithCandidateGenerationInfo ], @Named(ModuleNames.ProducerBasedUnifiedSimilarityEngine) producerBasedUnifiedSimilarityEngine: StandardSimilarityEngine[ ProducerBasedUnifiedSimilarityEngine.Query, TweetWithCandidateGenerationInfo ], @Named(ModuleNames.ConsumerEmbeddingBasedTripSimilarityEngine) consumerEmbeddingBasedTripSimilarityEngine: StandardSimilarityEngine[ TripEngineQuery, TripTweetWithScore ], @Named(ModuleNames.ConsumerEmbeddingBasedTwHINANNSimilarityEngine) consumerBasedTwHINANNSimilarityEngine: HnswANNSimilarityEngine, @Named(ModuleNames.ConsumerEmbeddingBasedTwoTowerANNSimilarityEngine) consumerBasedTwoTowerSimilarityEngine: HnswANNSimilarityEngine, @Named(ModuleNames.ConsumersBasedUserVideoGraphSimilarityEngine) consumersBasedUserVideoGraphSimilarityEngine: StandardSimilarityEngine[ ConsumersBasedUserVideoGraphSimilarityEngine.Query, TweetWithScore ], @Named(ModuleNames.UserTweetEntityGraphSimilarityEngine) userTweetEntityGraphSimilarityEngine: StandardSimilarityEngine[ UserTweetEntityGraphSimilarityEngine.Query, TweetWithScoreAndSocialProof ], @Named(ModuleNames.ConsumerBasedWalsSimilarityEngine) consumerBasedWalsSimilarityEngine: StandardSimilarityEngine[ ConsumerBasedWalsSimilarityEngine.Query, TweetWithScore ], tweetInfoStore: ReadableStore[TweetId, TweetInfo], globalStats: StatsReceiver, ) { import CandidateSourcesRouter._ val stats: StatsReceiver = globalStats.scope(this.getClass.getSimpleName) def fetchCandidates( requestUserId: UserId, sourceSignals: Set[SourceInfo], sourceGraphs: Map[String, Option[GraphSourceInfo]], params: configapi.Params, ): Future[Seq[Seq[InitialCandidate]]] = { val tweetBasedCandidatesFuture = getCandidates( getTweetBasedSourceInfo(sourceSignals), params, TweetBasedUnifiedSimilarityEngine.fromParams, tweetBasedUnifiedSimilarityEngine.getCandidates) val producerBasedCandidatesFuture = getCandidates( getProducerBasedSourceInfo(sourceSignals), params, ProducerBasedUnifiedSimilarityEngine.fromParams(_, _), producerBasedUnifiedSimilarityEngine.getCandidates ) val simClustersInterestedInBasedCandidatesFuture = getCandidatesPerSimilarityEngineModel( requestUserId, params, SimClustersInterestedInCandidateGeneration.fromParams, simClustersInterestedInCandidateGeneration.get) val consumerEmbeddingBasedLogFavBasedTripCandidatesFuture = if (params( ConsumerEmbeddingBasedCandidateGenerationParams.EnableLogFavBasedSimClustersTripParam)) { getSimClustersTripCandidates( params, ConsumerEmbeddingBasedTripSimilarityEngine.fromParams( ModelConfig.ConsumerLogFavBasedInterestedInEmbedding, InternalId.UserId(requestUserId), params ), consumerEmbeddingBasedTripSimilarityEngine ).map { Seq(_) } } else Future.Nil val consumersBasedUvgRealGraphInCandidatesFuture = if (params(ConsumersBasedUserVideoGraphParams.EnableSourceParam)) { val realGraphInGraphSourceInfoOpt = getGraphSourceInfoBySourceType(SourceType.RealGraphIn.name, sourceGraphs) getGraphBasedCandidates( params, ConsumersBasedUserVideoGraphSimilarityEngine .fromParamsForRealGraphIn( realGraphInGraphSourceInfoOpt .map { graphSourceInfo => graphSourceInfo.seedWithScores }.getOrElse(Map.empty), params), consumersBasedUserVideoGraphSimilarityEngine, ConsumersBasedUserVideoGraphSimilarityEngine.toSimilarityEngineInfo, realGraphInGraphSourceInfoOpt ).map { Seq(_) } } else Future.Nil val consumerEmbeddingBasedFollowBasedTripCandidatesFuture = if (params( ConsumerEmbeddingBasedCandidateGenerationParams.EnableFollowBasedSimClustersTripParam)) { getSimClustersTripCandidates( params, ConsumerEmbeddingBasedTripSimilarityEngine.fromParams( ModelConfig.ConsumerFollowBasedInterestedInEmbedding, InternalId.UserId(requestUserId), params ), consumerEmbeddingBasedTripSimilarityEngine ).map { Seq(_) } } else Future.Nil val consumerBasedWalsCandidatesFuture = if (params( ConsumerBasedWalsParams.EnableSourceParam )) { getConsumerBasedWalsCandidates(sourceSignals, params) }.map { Seq(_) } else Future.Nil val consumerEmbeddingBasedTwHINCandidatesFuture = if (params(ConsumerEmbeddingBasedCandidateGenerationParams.EnableTwHINParam)) { getHnswCandidates( params, ConsumerEmbeddingBasedTwHINSimilarityEngine.fromParams( InternalId.UserId(requestUserId), params), consumerBasedTwHINANNSimilarityEngine ).map { Seq(_) } } else Future.Nil val consumerEmbeddingBasedTwoTowerCandidatesFuture = if (params(ConsumerEmbeddingBasedCandidateGenerationParams.EnableTwoTowerParam)) { getHnswCandidates( params, ConsumerEmbeddingBasedTwoTowerSimilarityEngine.fromParams( InternalId.UserId(requestUserId), params), consumerBasedTwoTowerSimilarityEngine ).map { Seq(_) } } else Future.Nil val customizedRetrievalBasedCandidatesFuture = getCandidatesPerSimilarityEngineModel( requestUserId, params, CustomizedRetrievalCandidateGeneration.fromParams, customizedRetrievalCandidateGeneration.get) Future .collect( Seq( tweetBasedCandidatesFuture, producerBasedCandidatesFuture, simClustersInterestedInBasedCandidatesFuture, consumerBasedWalsCandidatesFuture, consumerEmbeddingBasedLogFavBasedTripCandidatesFuture, consumerEmbeddingBasedFollowBasedTripCandidatesFuture, consumerEmbeddingBasedTwHINCandidatesFuture, consumerEmbeddingBasedTwoTowerCandidatesFuture, consumersBasedUvgRealGraphInCandidatesFuture, customizedRetrievalBasedCandidatesFuture )).map { candidatesList => // remove empty innerSeq val result = candidatesList.flatten.filter(_.nonEmpty) stats.stat("numOfSequences").add(result.size) stats.stat("flattenCandidatesWithDup").add(result.flatten.size) result } } private def getGraphBasedCandidates[QueryType]( params: configapi.Params, query: EngineQuery[QueryType], engine: StandardSimilarityEngine[QueryType, TweetWithScore], toSimilarityEngineInfo: Double => SimilarityEngineInfo, graphSourceInfoOpt: Option[GraphSourceInfo] = None ): Future[Seq[InitialCandidate]] = { val candidatesOptFut = engine.getCandidates(query) val tweetsWithCandidateGenerationInfoOptFut = candidatesOptFut.map { _.map { tweetsWithScores => val sortedCandidates = tweetsWithScores.sortBy(-_.score) engine.getScopedStats.stat("sortedCandidates_size").add(sortedCandidates.size) val tweetsWithCandidateGenerationInfo = sortedCandidates.map { tweetWithScore => { val similarityEngineInfo = toSimilarityEngineInfo(tweetWithScore.score) val sourceInfo = graphSourceInfoOpt.map { graphSourceInfo => // The internalId is a placeholder value. We do not plan to store the full seedUserId set. SourceInfo( sourceType = graphSourceInfo.sourceType, internalId = InternalId.UserId(0L), sourceEventTime = None ) } TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( sourceInfo, similarityEngineInfo, Seq.empty // Atomic Similarity Engine. Hence it has no contributing SEs ) ) } } val maxCandidateNum = params(GlobalParams.MaxCandidateNumPerSourceKeyParam) tweetsWithCandidateGenerationInfo.take(maxCandidateNum) } } for { tweetsWithCandidateGenerationInfoOpt <- tweetsWithCandidateGenerationInfoOptFut initialCandidates <- convertToInitialCandidates( tweetsWithCandidateGenerationInfoOpt.toSeq.flatten) } yield initialCandidates } private def getCandidates[QueryType]( sourceSignals: Set[SourceInfo], params: configapi.Params, fromParams: (SourceInfo, configapi.Params) => QueryType, getFunc: QueryType => Future[Option[Seq[TweetWithCandidateGenerationInfo]]] ): Future[Seq[Seq[InitialCandidate]]] = { val queries = sourceSignals.map { sourceInfo => fromParams(sourceInfo, params) }.toSeq Future .collect { queries.map { query => for { candidates <- getFunc(query) prefilterCandidates <- convertToInitialCandidates(candidates.toSeq.flatten) } yield { prefilterCandidates } } } } private def getConsumerBasedWalsCandidates( sourceSignals: Set[SourceInfo], params: configapi.Params ): Future[Seq[InitialCandidate]] = { // Fetch source signals and filter them based on age. val signals = FilterUtil.tweetSourceAgeFilter( getConsumerBasedWalsSourceInfo(sourceSignals).toSeq, params(ConsumerBasedWalsParams.MaxTweetSignalAgeHoursParam)) val candidatesOptFut = consumerBasedWalsSimilarityEngine.getCandidates( ConsumerBasedWalsSimilarityEngine.fromParams(signals, params) ) val tweetsWithCandidateGenerationInfoOptFut = candidatesOptFut.map { _.map { tweetsWithScores => val sortedCandidates = tweetsWithScores.sortBy(-_.score) val filteredCandidates = FilterUtil.tweetAgeFilter(sortedCandidates, params(GlobalParams.MaxTweetAgeHoursParam)) consumerBasedWalsSimilarityEngine.getScopedStats .stat("filteredCandidates_size").add(filteredCandidates.size) val tweetsWithCandidateGenerationInfo = filteredCandidates.map { tweetWithScore => { val similarityEngineInfo = ConsumerBasedWalsSimilarityEngine.toSimilarityEngineInfo(tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( None, similarityEngineInfo, Seq.empty // Atomic Similarity Engine. Hence it has no contributing SEs ) ) } } val maxCandidateNum = params(GlobalParams.MaxCandidateNumPerSourceKeyParam) tweetsWithCandidateGenerationInfo.take(maxCandidateNum) } } for { tweetsWithCandidateGenerationInfoOpt <- tweetsWithCandidateGenerationInfoOptFut initialCandidates <- convertToInitialCandidates( tweetsWithCandidateGenerationInfoOpt.toSeq.flatten) } yield initialCandidates } private def getSimClustersTripCandidates( params: configapi.Params, query: TripEngineQuery, engine: StandardSimilarityEngine[ TripEngineQuery, TripTweetWithScore ], ): Future[Seq[InitialCandidate]] = { val tweetsWithCandidatesGenerationInfoOptFut = engine.getCandidates(EngineQuery(query, params)).map { _.map { _.map { tweetWithScore => // define filters TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( None, SimilarityEngineInfo( SimilarityEngineType.ExploreTripOfflineSimClustersTweets, None, Some(tweetWithScore.score)), Seq.empty ) ) } } } for { tweetsWithCandidateGenerationInfoOpt <- tweetsWithCandidatesGenerationInfoOptFut initialCandidates <- convertToInitialCandidates( tweetsWithCandidateGenerationInfoOpt.toSeq.flatten) } yield initialCandidates } private def getHnswCandidates( params: configapi.Params, query: HnswANNEngineQuery, engine: HnswANNSimilarityEngine, ): Future[Seq[InitialCandidate]] = { val candidatesOptFut = engine.getCandidates(query) val tweetsWithCandidateGenerationInfoOptFut = candidatesOptFut.map { _.map { tweetsWithScores => val sortedCandidates = tweetsWithScores.sortBy(-_.score) val filteredCandidates = FilterUtil.tweetAgeFilter(sortedCandidates, params(GlobalParams.MaxTweetAgeHoursParam)) engine.getScopedStats.stat("filteredCandidates_size").add(filteredCandidates.size) val tweetsWithCandidateGenerationInfo = filteredCandidates.map { tweetWithScore => { val similarityEngineInfo = engine.toSimilarityEngineInfo(query, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( None, similarityEngineInfo, Seq.empty // Atomic Similarity Engine. Hence it has no contributing SEs ) ) } } val maxCandidateNum = params(GlobalParams.MaxCandidateNumPerSourceKeyParam) tweetsWithCandidateGenerationInfo.take(maxCandidateNum) } } for { tweetsWithCandidateGenerationInfoOpt <- tweetsWithCandidateGenerationInfoOptFut initialCandidates <- convertToInitialCandidates( tweetsWithCandidateGenerationInfoOpt.toSeq.flatten) } yield initialCandidates } /** * Returns candidates from each similarity engine separately. * For 1 requestUserId, it will fetch results from each similarity engine e_i, * and returns Seq[Seq[TweetCandidate]]. */ private def getCandidatesPerSimilarityEngineModel[QueryType]( requestUserId: UserId, params: configapi.Params, fromParams: (InternalId, configapi.Params) => QueryType, getFunc: QueryType => Future[ Option[Seq[Seq[TweetWithCandidateGenerationInfo]]] ] ): Future[Seq[Seq[InitialCandidate]]] = { val query = fromParams(InternalId.UserId(requestUserId), params) getFunc(query).flatMap { candidatesPerSimilarityEngineModelOpt => val candidatesPerSimilarityEngineModel = candidatesPerSimilarityEngineModelOpt.toSeq.flatten Future.collect { candidatesPerSimilarityEngineModel.map(convertToInitialCandidates) } } } private[candidate_generation] def convertToInitialCandidates( candidates: Seq[TweetWithCandidateGenerationInfo], ): Future[Seq[InitialCandidate]] = { val tweetIds = candidates.map(_.tweetId).toSet Future.collect(tweetInfoStore.multiGet(tweetIds)).map { tweetInfos => /*** * If tweetInfo does not exist, we will filter out this tweet candidate. */ candidates.collect { case candidate if tweetInfos.getOrElse(candidate.tweetId, None).isDefined => val tweetInfo = tweetInfos(candidate.tweetId) .getOrElse(throw new IllegalStateException("Check previous line's condition")) InitialCandidate( tweetId = candidate.tweetId, tweetInfo = tweetInfo, candidate.candidateGenerationInfo ) } } } } object CandidateSourcesRouter { def getGraphSourceInfoBySourceType( sourceTypeStr: String, sourceGraphs: Map[String, Option[GraphSourceInfo]] ): Option[GraphSourceInfo] = { sourceGraphs.getOrElse(sourceTypeStr, None) } def getTweetBasedSourceInfo( sourceSignals: Set[SourceInfo] ): Set[SourceInfo] = { sourceSignals.collect { case sourceInfo if AllowedSourceTypesForTweetBasedUnifiedSE.contains(sourceInfo.sourceType.value) => sourceInfo } } def getProducerBasedSourceInfo( sourceSignals: Set[SourceInfo] ): Set[SourceInfo] = { sourceSignals.collect { case sourceInfo if AllowedSourceTypesForProducerBasedUnifiedSE.contains(sourceInfo.sourceType.value) => sourceInfo } } def getConsumerBasedWalsSourceInfo( sourceSignals: Set[SourceInfo] ): Set[SourceInfo] = { sourceSignals.collect { case sourceInfo if AllowedSourceTypesForConsumerBasedWalsSE.contains(sourceInfo.sourceType.value) => sourceInfo } } /*** * Signal funneling should not exist in CG or even in any SimilarityEngine. * They will be in Router, or eventually, in CrCandidateGenerator. */ val AllowedSourceTypesForConsumerBasedWalsSE = Set( SourceType.TweetFavorite.value, SourceType.Retweet.value, SourceType.TweetDontLike.value, //currently no-op SourceType.TweetReport.value, //currently no-op SourceType.AccountMute.value, //currently no-op SourceType.AccountBlock.value //currently no-op ) val AllowedSourceTypesForTweetBasedUnifiedSE = Set( SourceType.TweetFavorite.value, SourceType.Retweet.value, SourceType.OriginalTweet.value, SourceType.Reply.value, SourceType.TweetShare.value, SourceType.NotificationClick.value, SourceType.GoodTweetClick.value, SourceType.VideoTweetQualityView.value, SourceType.VideoTweetPlayback50.value, SourceType.TweetAggregation.value, ) val AllowedSourceTypesForProducerBasedUnifiedSE = Set( SourceType.UserFollow.value, SourceType.UserRepeatedProfileVisit.value, SourceType.RealGraphOon.value, SourceType.FollowRecommendation.value, SourceType.UserTrafficAttributionProfileVisit.value, SourceType.GoodProfileClick.value, SourceType.ProducerAggregation.value, ) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation/CrCandidateGenerator.scala ================================================ package com.twitter.cr_mixer.candidate_generation import com.twitter.cr_mixer.blender.SwitchBlender import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.filter.PostRankFilterRunner import com.twitter.cr_mixer.filter.PreRankFilterRunner import com.twitter.cr_mixer.logging.CrMixerScribeLogger import com.twitter.cr_mixer.model.BlendedCandidate import com.twitter.cr_mixer.model.CrCandidateGeneratorQuery import com.twitter.cr_mixer.model.GraphSourceInfo import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.model.RankedCandidate import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.param.RankerParams import com.twitter.cr_mixer.param.RecentNegativeSignalParams import com.twitter.cr_mixer.ranker.SwitchRanker import com.twitter.cr_mixer.source_signal.SourceInfoRouter import com.twitter.cr_mixer.source_signal.UssStore.EnabledNegativeSourceTypes import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.util.Future import com.twitter.util.JavaTimer import com.twitter.util.Timer import javax.inject.Inject import javax.inject.Singleton /** * For now it performs the main steps as follows: * 1. Source signal (via USS, FRS) fetch * 2. Candidate generation * 3. Filtering * 4. Interleave blender * 5. Ranker * 6. Post-ranker filter * 7. Truncation */ @Singleton class CrCandidateGenerator @Inject() ( sourceInfoRouter: SourceInfoRouter, candidateSourceRouter: CandidateSourcesRouter, switchBlender: SwitchBlender, preRankFilterRunner: PreRankFilterRunner, postRankFilterRunner: PostRankFilterRunner, switchRanker: SwitchRanker, crMixerScribeLogger: CrMixerScribeLogger, timeoutConfig: TimeoutConfig, globalStats: StatsReceiver) { private val timer: Timer = new JavaTimer(true) private val stats: StatsReceiver = globalStats.scope(this.getClass.getCanonicalName) private val fetchSourcesStats = stats.scope("fetchSources") private val fetchPositiveSourcesStats = stats.scope("fetchPositiveSources") private val fetchNegativeSourcesStats = stats.scope("fetchNegativeSources") private val fetchCandidatesStats = stats.scope("fetchCandidates") private val fetchCandidatesAfterFilterStats = stats.scope("fetchCandidatesAfterFilter") private val preRankFilterStats = stats.scope("preRankFilter") private val interleaveStats = stats.scope("interleave") private val rankStats = stats.scope("rank") private val postRankFilterStats = stats.scope("postRankFilter") private val blueVerifiedTweetStats = stats.scope("blueVerifiedTweetStats") private val blueVerifiedTweetStatsPerSimilarityEngine = stats.scope("blueVerifiedTweetStatsPerSimilarityEngine") def get(query: CrCandidateGeneratorQuery): Future[Seq[RankedCandidate]] = { val allStats = stats.scope("all") val perProductStats = stats.scope("perProduct", query.product.toString) val perProductBlueVerifiedStats = blueVerifiedTweetStats.scope("perProduct", query.product.toString) StatsUtil.trackItemsStats(allStats) { trackResultStats(perProductStats) { StatsUtil.trackItemsStats(perProductStats) { val result = for { (sourceSignals, sourceGraphsMap) <- StatsUtil.trackBlockStats(fetchSourcesStats) { fetchSources(query) } initialCandidates <- StatsUtil.trackBlockStats(fetchCandidatesAfterFilterStats) { // find the positive and negative signals val (positiveSignals, negativeSignals) = sourceSignals.partition { signal => !EnabledNegativeSourceTypes.contains(signal.sourceType) } fetchPositiveSourcesStats.stat("size").add(positiveSignals.size) fetchNegativeSourcesStats.stat("size").add(negativeSignals.size) // find the positive signals to keep, removing block and muted users val filteredSourceInfo = if (negativeSignals.nonEmpty && query.params( RecentNegativeSignalParams.EnableSourceParam)) { filterSourceInfo(positiveSignals, negativeSignals) } else { positiveSignals } // fetch candidates from the positive signals StatsUtil.trackBlockStats(fetchCandidatesStats) { fetchCandidates(query, filteredSourceInfo, sourceGraphsMap) } } filteredCandidates <- StatsUtil.trackBlockStats(preRankFilterStats) { preRankFilter(query, initialCandidates) } interleavedCandidates <- StatsUtil.trackItemsStats(interleaveStats) { interleave(query, filteredCandidates) } rankedCandidates <- StatsUtil.trackItemsStats(rankStats) { val candidatesToRank = interleavedCandidates.take(query.params(RankerParams.MaxCandidatesToRank)) rank(query, candidatesToRank) } postRankFilterCandidates <- StatsUtil.trackItemsStats(postRankFilterStats) { postRankFilter(query, rankedCandidates) } } yield { trackTopKStats( 800, postRankFilterCandidates, isQueryK = false, perProductBlueVerifiedStats) trackTopKStats( 400, postRankFilterCandidates, isQueryK = false, perProductBlueVerifiedStats) trackTopKStats( query.maxNumResults, postRankFilterCandidates, isQueryK = true, perProductBlueVerifiedStats) val (blueVerifiedTweets, remainingTweets) = postRankFilterCandidates.partition( _.tweetInfo.hasBlueVerifiedAnnotation.contains(true)) val topKBlueVerified = blueVerifiedTweets.take(query.maxNumResults) val topKRemaining = remainingTweets.take(query.maxNumResults - topKBlueVerified.size) trackBlueVerifiedTweetStats(topKBlueVerified, perProductBlueVerifiedStats) if (topKBlueVerified.nonEmpty && query.params(RankerParams.EnableBlueVerifiedTopK)) { topKBlueVerified ++ topKRemaining } else { postRankFilterCandidates } } result.raiseWithin(timeoutConfig.serviceTimeout)(timer) } } } } private def fetchSources( query: CrCandidateGeneratorQuery ): Future[(Set[SourceInfo], Map[String, Option[GraphSourceInfo]])] = { crMixerScribeLogger.scribeSignalSources( query, sourceInfoRouter .get(query.userId, query.product, query.userState, query.params)) } private def filterSourceInfo( positiveSignals: Set[SourceInfo], negativeSignals: Set[SourceInfo] ): Set[SourceInfo] = { val filterUsers: Set[Long] = negativeSignals.flatMap { case SourceInfo(_, InternalId.UserId(userId), _) => Some(userId) case _ => None } positiveSignals.filter { case SourceInfo(_, InternalId.UserId(userId), _) => !filterUsers.contains(userId) case _ => true } } def fetchCandidates( query: CrCandidateGeneratorQuery, sourceSignals: Set[SourceInfo], sourceGraphs: Map[String, Option[GraphSourceInfo]] ): Future[Seq[Seq[InitialCandidate]]] = { val initialCandidates = candidateSourceRouter .fetchCandidates( query.userId, sourceSignals, sourceGraphs, query.params ) initialCandidates.map(_.flatten.map { candidate => if (candidate.tweetInfo.hasBlueVerifiedAnnotation.contains(true)) { blueVerifiedTweetStatsPerSimilarityEngine .scope(query.product.toString).scope( candidate.candidateGenerationInfo.contributingSimilarityEngines.head.similarityEngineType.toString).counter( candidate.tweetInfo.authorId.toString).incr() } }) crMixerScribeLogger.scribeInitialCandidates( query, initialCandidates ) } private def preRankFilter( query: CrCandidateGeneratorQuery, candidates: Seq[Seq[InitialCandidate]] ): Future[Seq[Seq[InitialCandidate]]] = { crMixerScribeLogger.scribePreRankFilterCandidates( query, preRankFilterRunner .runSequentialFilters(query, candidates)) } private def postRankFilter( query: CrCandidateGeneratorQuery, candidates: Seq[RankedCandidate] ): Future[Seq[RankedCandidate]] = { postRankFilterRunner.run(query, candidates) } private def interleave( query: CrCandidateGeneratorQuery, candidates: Seq[Seq[InitialCandidate]] ): Future[Seq[BlendedCandidate]] = { crMixerScribeLogger.scribeInterleaveCandidates( query, switchBlender .blend(query.params, query.userState, candidates)) } private def rank( query: CrCandidateGeneratorQuery, candidates: Seq[BlendedCandidate], ): Future[Seq[RankedCandidate]] = { crMixerScribeLogger.scribeRankedCandidates( query, switchRanker.rank(query, candidates) ) } private def trackResultStats( stats: StatsReceiver )( fn: => Future[Seq[RankedCandidate]] ): Future[Seq[RankedCandidate]] = { fn.onSuccess { candidates => trackReasonChosenSourceTypeStats(candidates, stats) trackReasonChosenSimilarityEngineStats(candidates, stats) trackPotentialReasonsSourceTypeStats(candidates, stats) trackPotentialReasonsSimilarityEngineStats(candidates, stats) } } private def trackReasonChosenSourceTypeStats( candidates: Seq[RankedCandidate], stats: StatsReceiver ): Unit = { candidates .groupBy(_.reasonChosen.sourceInfoOpt.map(_.sourceType)) .foreach { case (sourceTypeOpt, rankedCands) => val sourceType = sourceTypeOpt.map(_.toString).getOrElse("RequesterId") // default stats.stat("reasonChosen", "sourceType", sourceType, "size").add(rankedCands.size) } } private def trackReasonChosenSimilarityEngineStats( candidates: Seq[RankedCandidate], stats: StatsReceiver ): Unit = { candidates .groupBy(_.reasonChosen.similarityEngineInfo.similarityEngineType) .foreach { case (seInfoType, rankedCands) => stats .stat("reasonChosen", "similarityEngine", seInfoType.toString, "size").add( rankedCands.size) } } private def trackPotentialReasonsSourceTypeStats( candidates: Seq[RankedCandidate], stats: StatsReceiver ): Unit = { candidates .flatMap(_.potentialReasons.map(_.sourceInfoOpt.map(_.sourceType))) .groupBy(source => source) .foreach { case (sourceInfoOpt, seq) => val sourceType = sourceInfoOpt.map(_.toString).getOrElse("RequesterId") // default stats.stat("potentialReasons", "sourceType", sourceType, "size").add(seq.size) } } private def trackPotentialReasonsSimilarityEngineStats( candidates: Seq[RankedCandidate], stats: StatsReceiver ): Unit = { candidates .flatMap(_.potentialReasons.map(_.similarityEngineInfo.similarityEngineType)) .groupBy(se => se) .foreach { case (seType, seq) => stats.stat("potentialReasons", "similarityEngine", seType.toString, "size").add(seq.size) } } private def trackBlueVerifiedTweetStats( candidates: Seq[RankedCandidate], statsReceiver: StatsReceiver ): Unit = { candidates.foreach { candidate => if (candidate.tweetInfo.hasBlueVerifiedAnnotation.contains(true)) { statsReceiver.counter(candidate.tweetInfo.authorId.toString).incr() statsReceiver .scope(candidate.tweetInfo.authorId.toString).counter(candidate.tweetId.toString).incr() } } } private def trackTopKStats( k: Int, tweetCandidates: Seq[RankedCandidate], isQueryK: Boolean, statsReceiver: StatsReceiver ): Unit = { val (topK, beyondK) = tweetCandidates.splitAt(k) val blueVerifiedIds = tweetCandidates.collect { case candidate if candidate.tweetInfo.hasBlueVerifiedAnnotation.contains(true) => candidate.tweetInfo.authorId }.toSet blueVerifiedIds.foreach { blueVerifiedId => val numTweetsTopK = topK.count(_.tweetInfo.authorId == blueVerifiedId) val numTweetsBeyondK = beyondK.count(_.tweetInfo.authorId == blueVerifiedId) if (isQueryK) { statsReceiver.scope(blueVerifiedId.toString).stat(s"topK").add(numTweetsTopK) statsReceiver .scope(blueVerifiedId.toString).stat(s"beyondK").add(numTweetsBeyondK) } else { statsReceiver.scope(blueVerifiedId.toString).stat(s"top$k").add(numTweetsTopK) statsReceiver .scope(blueVerifiedId.toString).stat(s"beyond$k").add(numTweetsBeyondK) } } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation/CustomizedRetrievalCandidateGeneration.scala ================================================ package com.twitter.cr_mixer.candidate_generation import com.twitter.cr_mixer.candidate_generation.CustomizedRetrievalCandidateGeneration.Query import com.twitter.cr_mixer.model.CandidateGenerationInfo import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithCandidateGenerationInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.CustomizedRetrievalBasedCandidateGenerationParams._ import com.twitter.cr_mixer.param.CustomizedRetrievalBasedTwhinParams._ import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.similarity_engine.DiffusionBasedSimilarityEngine import com.twitter.cr_mixer.similarity_engine.LookupEngineQuery import com.twitter.cr_mixer.similarity_engine.LookupSimilarityEngine import com.twitter.cr_mixer.similarity_engine.TwhinCollabFilterSimilarityEngine import com.twitter.cr_mixer.util.InterleaveUtil import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.base.CandidateSource import com.twitter.frigate.common.base.Stats import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.snowflake.id.SnowflakeId import com.twitter.timelines.configapi import com.twitter.util.Duration import com.twitter.util.Future import com.twitter.util.Time import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.mutable.ArrayBuffer /** * A candidate generator that fetches similar tweets from multiple customized retrieval based candidate sources * * Different from [[TweetBasedCandidateGeneration]], this store returns candidates from different * similarity engines without blending. In other words, this class shall not be thought of as a * Unified Similarity Engine. It is a CG that calls multiple singular Similarity Engines. */ @Singleton case class CustomizedRetrievalCandidateGeneration @Inject() ( @Named(ModuleNames.TwhinCollabFilterSimilarityEngine) twhinCollabFilterSimilarityEngine: LookupSimilarityEngine[ TwhinCollabFilterSimilarityEngine.Query, TweetWithScore ], @Named(ModuleNames.DiffusionBasedSimilarityEngine) diffusionBasedSimilarityEngine: LookupSimilarityEngine[ DiffusionBasedSimilarityEngine.Query, TweetWithScore ], statsReceiver: StatsReceiver) extends CandidateSource[ Query, Seq[TweetWithCandidateGenerationInfo] ] { override def name: String = this.getClass.getSimpleName private val stats = statsReceiver.scope(name) private val fetchCandidatesStat = stats.scope("fetchCandidates") /** * For each Similarity Engine Model, return a list of tweet candidates */ override def get( query: Query ): Future[Option[Seq[Seq[TweetWithCandidateGenerationInfo]]]] = { query.internalId match { case InternalId.UserId(_) => Stats.trackOption(fetchCandidatesStat) { val twhinCollabFilterForFollowCandidatesFut = if (query.enableTwhinCollabFilter) { twhinCollabFilterSimilarityEngine.getCandidates(query.twhinCollabFilterFollowQuery) } else Future.None val twhinCollabFilterForEngagementCandidatesFut = if (query.enableTwhinCollabFilter) { twhinCollabFilterSimilarityEngine.getCandidates( query.twhinCollabFilterEngagementQuery) } else Future.None val twhinMultiClusterForFollowCandidatesFut = if (query.enableTwhinMultiCluster) { twhinCollabFilterSimilarityEngine.getCandidates(query.twhinMultiClusterFollowQuery) } else Future.None val twhinMultiClusterForEngagementCandidatesFut = if (query.enableTwhinMultiCluster) { twhinCollabFilterSimilarityEngine.getCandidates( query.twhinMultiClusterEngagementQuery) } else Future.None val diffusionBasedSimilarityEngineCandidatesFut = if (query.enableRetweetBasedDiffusion) { diffusionBasedSimilarityEngine.getCandidates(query.diffusionBasedSimilarityEngineQuery) } else Future.None Future .join( twhinCollabFilterForFollowCandidatesFut, twhinCollabFilterForEngagementCandidatesFut, twhinMultiClusterForFollowCandidatesFut, twhinMultiClusterForEngagementCandidatesFut, diffusionBasedSimilarityEngineCandidatesFut ).map { case ( twhinCollabFilterForFollowCandidates, twhinCollabFilterForEngagementCandidates, twhinMultiClusterForFollowCandidates, twhinMultiClusterForEngagementCandidates, diffusionBasedSimilarityEngineCandidates) => val maxCandidateNumPerSourceKey = 200 val twhinCollabFilterForFollowWithCGInfo = getTwhinCollabCandidatesWithCGInfo( twhinCollabFilterForFollowCandidates, maxCandidateNumPerSourceKey, query.twhinCollabFilterFollowQuery, ) val twhinCollabFilterForEngagementWithCGInfo = getTwhinCollabCandidatesWithCGInfo( twhinCollabFilterForEngagementCandidates, maxCandidateNumPerSourceKey, query.twhinCollabFilterEngagementQuery, ) val twhinMultiClusterForFollowWithCGInfo = getTwhinCollabCandidatesWithCGInfo( twhinMultiClusterForFollowCandidates, maxCandidateNumPerSourceKey, query.twhinMultiClusterFollowQuery, ) val twhinMultiClusterForEngagementWithCGInfo = getTwhinCollabCandidatesWithCGInfo( twhinMultiClusterForEngagementCandidates, maxCandidateNumPerSourceKey, query.twhinMultiClusterEngagementQuery, ) val retweetBasedDiffusionWithCGInfo = getDiffusionBasedCandidatesWithCGInfo( diffusionBasedSimilarityEngineCandidates, maxCandidateNumPerSourceKey, query.diffusionBasedSimilarityEngineQuery, ) val twhinCollabCandidateSourcesToBeInterleaved = ArrayBuffer[Seq[TweetWithCandidateGenerationInfo]]( twhinCollabFilterForFollowWithCGInfo, twhinCollabFilterForEngagementWithCGInfo, ) val twhinMultiClusterCandidateSourcesToBeInterleaved = ArrayBuffer[Seq[TweetWithCandidateGenerationInfo]]( twhinMultiClusterForFollowWithCGInfo, twhinMultiClusterForEngagementWithCGInfo, ) val interleavedTwhinCollabCandidates = InterleaveUtil.interleave(twhinCollabCandidateSourcesToBeInterleaved) val interleavedTwhinMultiClusterCandidates = InterleaveUtil.interleave(twhinMultiClusterCandidateSourcesToBeInterleaved) val twhinCollabFilterResults = if (interleavedTwhinCollabCandidates.nonEmpty) { Some(interleavedTwhinCollabCandidates.take(maxCandidateNumPerSourceKey)) } else None val twhinMultiClusterResults = if (interleavedTwhinMultiClusterCandidates.nonEmpty) { Some(interleavedTwhinMultiClusterCandidates.take(maxCandidateNumPerSourceKey)) } else None val diffusionResults = if (retweetBasedDiffusionWithCGInfo.nonEmpty) { Some(retweetBasedDiffusionWithCGInfo.take(maxCandidateNumPerSourceKey)) } else None Some( Seq( twhinCollabFilterResults, twhinMultiClusterResults, diffusionResults ).flatten) } } case _ => throw new IllegalArgumentException("sourceId_is_not_userId_cnt") } } /** Returns a list of tweets that are generated less than `maxTweetAgeHours` hours ago */ private def tweetAgeFilter( candidates: Seq[TweetWithScore], maxTweetAgeHours: Duration ): Seq[TweetWithScore] = { // Tweet IDs are approximately chronological (see http://go/snowflake), // so we are building the earliest tweet id once // The per-candidate logic here then be candidate.tweetId > earliestPermittedTweetId, which is far cheaper. val earliestTweetId = SnowflakeId.firstIdFor(Time.now - maxTweetAgeHours) candidates.filter { candidate => candidate.tweetId >= earliestTweetId } } /** * AgeFilters tweetCandidates with stats * Only age filter logic is effective here (through tweetAgeFilter). This function acts mostly for metric logging. */ private def ageFilterWithStats( offlineInterestedInCandidates: Seq[TweetWithScore], maxTweetAgeHours: Duration, scopedStatsReceiver: StatsReceiver ): Seq[TweetWithScore] = { scopedStatsReceiver.stat("size").add(offlineInterestedInCandidates.size) val candidates = offlineInterestedInCandidates.map { candidate => TweetWithScore(candidate.tweetId, candidate.score) } val filteredCandidates = tweetAgeFilter(candidates, maxTweetAgeHours) scopedStatsReceiver.stat(f"filtered_size").add(filteredCandidates.size) if (filteredCandidates.isEmpty) scopedStatsReceiver.counter(f"empty").incr() filteredCandidates } private def getTwhinCollabCandidatesWithCGInfo( tweetCandidates: Option[Seq[TweetWithScore]], maxCandidateNumPerSourceKey: Int, twhinCollabFilterQuery: LookupEngineQuery[ TwhinCollabFilterSimilarityEngine.Query ], ): Seq[TweetWithCandidateGenerationInfo] = { val twhinTweets = tweetCandidates match { case Some(tweetsWithScores) => tweetsWithScores.map { tweetWithScore => TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( None, TwhinCollabFilterSimilarityEngine .toSimilarityEngineInfo(twhinCollabFilterQuery, tweetWithScore.score), Seq.empty ) ) } case _ => Seq.empty } twhinTweets.take(maxCandidateNumPerSourceKey) } private def getDiffusionBasedCandidatesWithCGInfo( tweetCandidates: Option[Seq[TweetWithScore]], maxCandidateNumPerSourceKey: Int, diffusionBasedSimilarityEngineQuery: LookupEngineQuery[ DiffusionBasedSimilarityEngine.Query ], ): Seq[TweetWithCandidateGenerationInfo] = { val diffusionTweets = tweetCandidates match { case Some(tweetsWithScores) => tweetsWithScores.map { tweetWithScore => TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( None, DiffusionBasedSimilarityEngine .toSimilarityEngineInfo(diffusionBasedSimilarityEngineQuery, tweetWithScore.score), Seq.empty ) ) } case _ => Seq.empty } diffusionTweets.take(maxCandidateNumPerSourceKey) } } object CustomizedRetrievalCandidateGeneration { case class Query( internalId: InternalId, maxCandidateNumPerSourceKey: Int, maxTweetAgeHours: Duration, // twhinCollabFilter enableTwhinCollabFilter: Boolean, twhinCollabFilterFollowQuery: LookupEngineQuery[ TwhinCollabFilterSimilarityEngine.Query ], twhinCollabFilterEngagementQuery: LookupEngineQuery[ TwhinCollabFilterSimilarityEngine.Query ], // twhinMultiCluster enableTwhinMultiCluster: Boolean, twhinMultiClusterFollowQuery: LookupEngineQuery[ TwhinCollabFilterSimilarityEngine.Query ], twhinMultiClusterEngagementQuery: LookupEngineQuery[ TwhinCollabFilterSimilarityEngine.Query ], enableRetweetBasedDiffusion: Boolean, diffusionBasedSimilarityEngineQuery: LookupEngineQuery[ DiffusionBasedSimilarityEngine.Query ], ) def fromParams( internalId: InternalId, params: configapi.Params ): Query = { val twhinCollabFilterFollowQuery = TwhinCollabFilterSimilarityEngine.fromParams( internalId, params(CustomizedRetrievalBasedTwhinCollabFilterFollowSource), params) val twhinCollabFilterEngagementQuery = TwhinCollabFilterSimilarityEngine.fromParams( internalId, params(CustomizedRetrievalBasedTwhinCollabFilterEngagementSource), params) val twhinMultiClusterFollowQuery = TwhinCollabFilterSimilarityEngine.fromParams( internalId, params(CustomizedRetrievalBasedTwhinMultiClusterFollowSource), params) val twhinMultiClusterEngagementQuery = TwhinCollabFilterSimilarityEngine.fromParams( internalId, params(CustomizedRetrievalBasedTwhinMultiClusterEngagementSource), params) val diffusionBasedSimilarityEngineQuery = DiffusionBasedSimilarityEngine.fromParams( internalId, params(CustomizedRetrievalBasedRetweetDiffusionSource), params) Query( internalId = internalId, maxCandidateNumPerSourceKey = params(GlobalParams.MaxCandidateNumPerSourceKeyParam), maxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam), // twhinCollabFilter enableTwhinCollabFilter = params(EnableTwhinCollabFilterClusterParam), twhinCollabFilterFollowQuery = twhinCollabFilterFollowQuery, twhinCollabFilterEngagementQuery = twhinCollabFilterEngagementQuery, enableTwhinMultiCluster = params(EnableTwhinMultiClusterParam), twhinMultiClusterFollowQuery = twhinMultiClusterFollowQuery, twhinMultiClusterEngagementQuery = twhinMultiClusterEngagementQuery, enableRetweetBasedDiffusion = params(EnableRetweetBasedDiffusionParam), diffusionBasedSimilarityEngineQuery = diffusionBasedSimilarityEngineQuery ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation/FrsTweetCandidateGenerator.scala ================================================ package com.twitter.cr_mixer.candidate_generation import com.twitter.contentrecommender.thriftscala.TweetInfo import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.FrsTweetCandidateGeneratorQuery import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithAuthor import com.twitter.cr_mixer.param.FrsParams import com.twitter.cr_mixer.similarity_engine.EarlybirdSimilarityEngineRouter import com.twitter.cr_mixer.source_signal.FrsStore import com.twitter.cr_mixer.source_signal.FrsStore.FrsQueryResult import com.twitter.cr_mixer.thriftscala.FrsTweet import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.util.DefaultTimer import com.twitter.frigate.common.util.StatsUtil import com.twitter.hermit.constants.AlgorithmFeedbackTokens import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap import com.twitter.hermit.model.Algorithm import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.common.UserId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi.Params import com.twitter.util.Future import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton /** * TweetCandidateGenerator based on FRS seed users. For now this candidate generator fetches seed * users from FRS, and retrieves the seed users' past tweets from Earlybird with Earlybird light * ranking models. */ @Singleton class FrsTweetCandidateGenerator @Inject() ( @Named(ModuleNames.FrsStore) frsStore: ReadableStore[FrsStore.Query, Seq[FrsQueryResult]], frsBasedSimilarityEngine: EarlybirdSimilarityEngineRouter, tweetInfoStore: ReadableStore[TweetId, TweetInfo], timeoutConfig: TimeoutConfig, globalStats: StatsReceiver) { import FrsTweetCandidateGenerator._ private val timer = DefaultTimer private val stats: StatsReceiver = globalStats.scope(this.getClass.getCanonicalName) private val fetchSeedsStats = stats.scope("fetchSeeds") private val fetchCandidatesStats = stats.scope("fetchCandidates") private val filterCandidatesStats = stats.scope("filterCandidates") private val hydrateCandidatesStats = stats.scope("hydrateCandidates") private val getCandidatesStats = stats.scope("getCandidates") /** * The function retrieves the candidate for the given user as follows: * 1. Seed user fetch from FRS. * 2. Candidate fetch from Earlybird. * 3. Filtering. * 4. Candidate hydration. * 5. Truncation. */ def get( frsTweetCandidateGeneratorQuery: FrsTweetCandidateGeneratorQuery ): Future[Seq[FrsTweet]] = { val userId = frsTweetCandidateGeneratorQuery.userId val product = frsTweetCandidateGeneratorQuery.product val allStats = stats.scope("all") val perProductStats = stats.scope("perProduct", product.name) StatsUtil.trackItemsStats(allStats) { StatsUtil.trackItemsStats(perProductStats) { val result = for { seedAuthorWithScores <- StatsUtil.trackOptionItemMapStats(fetchSeedsStats) { fetchSeeds( userId, frsTweetCandidateGeneratorQuery.impressedUserList, frsTweetCandidateGeneratorQuery.languageCodeOpt, frsTweetCandidateGeneratorQuery.countryCodeOpt, frsTweetCandidateGeneratorQuery.params, ) } tweetCandidates <- StatsUtil.trackOptionItemsStats(fetchCandidatesStats) { fetchCandidates( userId, seedAuthorWithScores.map(_.keys.toSeq).getOrElse(Seq.empty), frsTweetCandidateGeneratorQuery.impressedTweetList, seedAuthorWithScores.map(_.mapValues(_.score)).getOrElse(Map.empty), frsTweetCandidateGeneratorQuery.params ) } filteredTweetCandidates <- StatsUtil.trackOptionItemsStats(filterCandidatesStats) { filterCandidates( tweetCandidates, frsTweetCandidateGeneratorQuery.params ) } hydratedTweetCandidates <- StatsUtil.trackOptionItemsStats(hydrateCandidatesStats) { hydrateCandidates( seedAuthorWithScores, filteredTweetCandidates ) } } yield { hydratedTweetCandidates .map(_.take(frsTweetCandidateGeneratorQuery.maxNumResults)).getOrElse(Seq.empty) } result.raiseWithin(timeoutConfig.frsBasedTweetEndpointTimeout)(timer) } } } /** * Fetch recommended seed users from FRS */ private def fetchSeeds( userId: UserId, userDenyList: Set[UserId], languageCodeOpt: Option[String], countryCodeOpt: Option[String], params: Params ): Future[Option[Map[UserId, FrsQueryResult]]] = { frsStore .get( FrsStore.Query( userId, params(FrsParams.FrsBasedCandidateGenerationMaxSeedsNumParam), params(FrsParams.FrsBasedCandidateGenerationDisplayLocationParam).displayLocation, userDenyList.toSeq, languageCodeOpt, countryCodeOpt )).map { _.map { seedAuthors => seedAuthors.map(user => user.userId -> user).toMap } } } /** * Fetch tweet candidates from Earlybird */ private def fetchCandidates( searcherUserId: UserId, seedAuthors: Seq[UserId], impressedTweetList: Set[TweetId], frsUserToScores: Map[UserId, Double], params: Params ): Future[Option[Seq[TweetWithAuthor]]] = { if (seedAuthors.nonEmpty) { // call earlybird val query = EarlybirdSimilarityEngineRouter.queryFromParams( Some(searcherUserId), seedAuthors, impressedTweetList, frsUserToScoresForScoreAdjustment = Some(frsUserToScores), params ) frsBasedSimilarityEngine.get(query) } else Future.None } /** * Filter candidates that do not pass visibility filter policy */ private def filterCandidates( candidates: Option[Seq[TweetWithAuthor]], params: Params ): Future[Option[Seq[TweetWithAuthor]]] = { val tweetIds = candidates.map(_.map(_.tweetId).toSet).getOrElse(Set.empty) if (params(FrsParams.FrsBasedCandidateGenerationEnableVisibilityFilteringParam)) Future .collect(tweetInfoStore.multiGet(tweetIds)).map { tweetInfos => candidates.map { // If tweetInfo does not exist, we will filter out this tweet candidate. _.filter(candidate => tweetInfos.getOrElse(candidate.tweetId, None).isDefined) } } else { Future.value(candidates) } } /** * Hydrate the candidates with the FRS candidate sources and scores */ private def hydrateCandidates( frsAuthorWithScores: Option[Map[UserId, FrsQueryResult]], candidates: Option[Seq[TweetWithAuthor]] ): Future[Option[Seq[FrsTweet]]] = { Future.value { candidates.map { _.map { tweetWithAuthor => val frsQueryResult = frsAuthorWithScores.flatMap(_.get(tweetWithAuthor.authorId)) FrsTweet( tweetId = tweetWithAuthor.tweetId, authorId = tweetWithAuthor.authorId, frsPrimarySource = frsQueryResult.flatMap(_.primarySource), frsAuthorScore = frsQueryResult.map(_.score), frsCandidateSourceScores = frsQueryResult.flatMap { result => result.sourceWithScores.map { _.collect { // see TokenStrToAlgorithmMap @ https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/hermit/hermit-core/src/main/scala/com/twitter/hermit/constants/AlgorithmFeedbackTokens.scala // see Algorithm @ https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/hermit/hermit-core/src/main/scala/com/twitter/hermit/model/Algorithm.scala case (candidateSourceAlgoStr, score) if AlgorithmFeedbackTokens.TokenStrToAlgorithmMap.contains( candidateSourceAlgoStr) => AlgorithmToFeedbackTokenMap.getOrElse( AlgorithmFeedbackTokens.TokenStrToAlgorithmMap .getOrElse(candidateSourceAlgoStr, DefaultAlgo), DefaultAlgoToken) -> score } } } ) } } } } } object FrsTweetCandidateGenerator { val DefaultAlgo: Algorithm.Value = Algorithm.Other // 9999 is the token for Algorithm.Other val DefaultAlgoToken: Int = AlgorithmToFeedbackTokenMap.getOrElse(DefaultAlgo, 9999) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation/RelatedTweetCandidateGenerator.scala ================================================ package com.twitter.cr_mixer.candidate_generation import com.twitter.contentrecommender.thriftscala.TweetInfo import com.twitter.cr_mixer.filter.PreRankFilterRunner import com.twitter.cr_mixer.logging.RelatedTweetScribeLogger import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.model.RelatedTweetCandidateGeneratorQuery import com.twitter.cr_mixer.model.TweetWithCandidateGenerationInfo import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.similarity_engine.ProducerBasedUnifiedSimilarityEngine import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.similarity_engine.TweetBasedUnifiedSimilarityEngine import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.util.Future import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class RelatedTweetCandidateGenerator @Inject() ( @Named(ModuleNames.TweetBasedUnifiedSimilarityEngine) tweetBasedUnifiedSimilarityEngine: StandardSimilarityEngine[ TweetBasedUnifiedSimilarityEngine.Query, TweetWithCandidateGenerationInfo ], @Named(ModuleNames.ProducerBasedUnifiedSimilarityEngine) producerBasedUnifiedSimilarityEngine: StandardSimilarityEngine[ ProducerBasedUnifiedSimilarityEngine.Query, TweetWithCandidateGenerationInfo ], preRankFilterRunner: PreRankFilterRunner, relatedTweetScribeLogger: RelatedTweetScribeLogger, tweetInfoStore: ReadableStore[TweetId, TweetInfo], globalStats: StatsReceiver) { private val stats: StatsReceiver = globalStats.scope(this.getClass.getCanonicalName) private val fetchCandidatesStats = stats.scope("fetchCandidates") private val preRankFilterStats = stats.scope("preRankFilter") def get( query: RelatedTweetCandidateGeneratorQuery ): Future[Seq[InitialCandidate]] = { val allStats = stats.scope("all") val perProductStats = stats.scope("perProduct", query.product.toString) StatsUtil.trackItemsStats(allStats) { StatsUtil.trackItemsStats(perProductStats) { for { initialCandidates <- StatsUtil.trackBlockStats(fetchCandidatesStats) { fetchCandidates(query) } filteredCandidates <- StatsUtil.trackBlockStats(preRankFilterStats) { preRankFilter(query, initialCandidates) } } yield { filteredCandidates.headOption .getOrElse( throw new UnsupportedOperationException( "RelatedTweetCandidateGenerator results invalid") ).take(query.maxNumResults) } } } } def fetchCandidates( query: RelatedTweetCandidateGeneratorQuery ): Future[Seq[Seq[InitialCandidate]]] = { relatedTweetScribeLogger.scribeInitialCandidates( query, query.internalId match { case InternalId.TweetId(_) => getCandidatesFromSimilarityEngine( query, TweetBasedUnifiedSimilarityEngine.fromParamsForRelatedTweet, tweetBasedUnifiedSimilarityEngine.getCandidates) case InternalId.UserId(_) => getCandidatesFromSimilarityEngine( query, ProducerBasedUnifiedSimilarityEngine.fromParamsForRelatedTweet, producerBasedUnifiedSimilarityEngine.getCandidates) case _ => throw new UnsupportedOperationException( "RelatedTweetCandidateGenerator gets invalid InternalId") } ) } /*** * fetch Candidates from TweetBased/ProducerBased Unified Similarity Engine, * and apply VF filter based on TweetInfoStore * To align with the downstream processing (filter, rank), we tend to return a Seq[Seq[InitialCandidate]] * instead of a Seq[Candidate] even though we only have a Seq in it. */ private def getCandidatesFromSimilarityEngine[QueryType]( query: RelatedTweetCandidateGeneratorQuery, fromParamsForRelatedTweet: (InternalId, configapi.Params) => QueryType, getFunc: QueryType => Future[Option[Seq[TweetWithCandidateGenerationInfo]]] ): Future[Seq[Seq[InitialCandidate]]] = { /*** * We wrap the query to be a Seq of queries for the Sim Engine to ensure evolvability of candidate generation * and as a result, it will return Seq[Seq[InitialCandidate]] */ val engineQueries = Seq(fromParamsForRelatedTweet(query.internalId, query.params)) Future .collect { engineQueries.map { query => for { candidates <- getFunc(query) prefilterCandidates <- convertToInitialCandidates( candidates.toSeq.flatten ) } yield prefilterCandidates } } } private def preRankFilter( query: RelatedTweetCandidateGeneratorQuery, candidates: Seq[Seq[InitialCandidate]] ): Future[Seq[Seq[InitialCandidate]]] = { relatedTweetScribeLogger.scribePreRankFilterCandidates( query, preRankFilterRunner .runSequentialFilters(query, candidates)) } private[candidate_generation] def convertToInitialCandidates( candidates: Seq[TweetWithCandidateGenerationInfo], ): Future[Seq[InitialCandidate]] = { val tweetIds = candidates.map(_.tweetId).toSet Future.collect(tweetInfoStore.multiGet(tweetIds)).map { tweetInfos => /*** * If tweetInfo does not exist, we will filter out this tweet candidate. * This tweetInfo filter also acts as the VF filter */ candidates.collect { case candidate if tweetInfos.getOrElse(candidate.tweetId, None).isDefined => val tweetInfo = tweetInfos(candidate.tweetId) .getOrElse(throw new IllegalStateException("Check previous line's condition")) InitialCandidate( tweetId = candidate.tweetId, tweetInfo = tweetInfo, candidate.candidateGenerationInfo ) } } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation/RelatedVideoTweetCandidateGenerator.scala ================================================ package com.twitter.cr_mixer.candidate_generation import com.twitter.contentrecommender.thriftscala.TweetInfo import com.twitter.cr_mixer.filter.PreRankFilterRunner import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.model.RelatedVideoTweetCandidateGeneratorQuery import com.twitter.cr_mixer.model.TweetWithCandidateGenerationInfo import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.similarity_engine.TweetBasedUnifiedSimilarityEngine import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.util.Future import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class RelatedVideoTweetCandidateGenerator @Inject() ( @Named(ModuleNames.TweetBasedUnifiedSimilarityEngine) tweetBasedUnifiedSimilarityEngine: StandardSimilarityEngine[ TweetBasedUnifiedSimilarityEngine.Query, TweetWithCandidateGenerationInfo ], preRankFilterRunner: PreRankFilterRunner, tweetInfoStore: ReadableStore[TweetId, TweetInfo], globalStats: StatsReceiver) { private val stats: StatsReceiver = globalStats.scope(this.getClass.getCanonicalName) private val fetchCandidatesStats = stats.scope("fetchCandidates") private val preRankFilterStats = stats.scope("preRankFilter") def get( query: RelatedVideoTweetCandidateGeneratorQuery ): Future[Seq[InitialCandidate]] = { val allStats = stats.scope("all") val perProductStats = stats.scope("perProduct", query.product.toString) StatsUtil.trackItemsStats(allStats) { StatsUtil.trackItemsStats(perProductStats) { for { initialCandidates <- StatsUtil.trackBlockStats(fetchCandidatesStats) { fetchCandidates(query) } filteredCandidates <- StatsUtil.trackBlockStats(preRankFilterStats) { preRankFilter(query, initialCandidates) } } yield { filteredCandidates.headOption .getOrElse( throw new UnsupportedOperationException( "RelatedVideoTweetCandidateGenerator results invalid") ).take(query.maxNumResults) } } } } def fetchCandidates( query: RelatedVideoTweetCandidateGeneratorQuery ): Future[Seq[Seq[InitialCandidate]]] = { query.internalId match { case InternalId.TweetId(_) => getCandidatesFromSimilarityEngine( query, TweetBasedUnifiedSimilarityEngine.fromParamsForRelatedVideoTweet, tweetBasedUnifiedSimilarityEngine.getCandidates) case _ => throw new UnsupportedOperationException( "RelatedVideoTweetCandidateGenerator gets invalid InternalId") } } /*** * fetch Candidates from TweetBased/ProducerBased Unified Similarity Engine, * and apply VF filter based on TweetInfoStore * To align with the downstream processing (filter, rank), we tend to return a Seq[Seq[InitialCandidate]] * instead of a Seq[Candidate] even though we only have a Seq in it. */ private def getCandidatesFromSimilarityEngine[QueryType]( query: RelatedVideoTweetCandidateGeneratorQuery, fromParamsForRelatedVideoTweet: (InternalId, configapi.Params) => QueryType, getFunc: QueryType => Future[Option[Seq[TweetWithCandidateGenerationInfo]]] ): Future[Seq[Seq[InitialCandidate]]] = { /*** * We wrap the query to be a Seq of queries for the Sim Engine to ensure evolvability of candidate generation * and as a result, it will return Seq[Seq[InitialCandidate]] */ val engineQueries = Seq(fromParamsForRelatedVideoTweet(query.internalId, query.params)) Future .collect { engineQueries.map { query => for { candidates <- getFunc(query) prefilterCandidates <- convertToInitialCandidates( candidates.toSeq.flatten ) } yield prefilterCandidates } } } private def preRankFilter( query: RelatedVideoTweetCandidateGeneratorQuery, candidates: Seq[Seq[InitialCandidate]] ): Future[Seq[Seq[InitialCandidate]]] = { preRankFilterRunner .runSequentialFilters(query, candidates) } private[candidate_generation] def convertToInitialCandidates( candidates: Seq[TweetWithCandidateGenerationInfo], ): Future[Seq[InitialCandidate]] = { val tweetIds = candidates.map(_.tweetId).toSet Future.collect(tweetInfoStore.multiGet(tweetIds)).map { tweetInfos => /*** * If tweetInfo does not exist, we will filter out this tweet candidate. * This tweetInfo filter also acts as the VF filter */ candidates.collect { case candidate if tweetInfos.getOrElse(candidate.tweetId, None).isDefined => val tweetInfo = tweetInfos(candidate.tweetId) .getOrElse(throw new IllegalStateException("Check previous line's condition")) InitialCandidate( tweetId = candidate.tweetId, tweetInfo = tweetInfo, candidate.candidateGenerationInfo ) } } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation/SimClustersInterestedInCandidateGeneration.scala ================================================ package com.twitter.cr_mixer.candidate_generation import com.twitter.cr_mixer.model.CandidateGenerationInfo import com.twitter.cr_mixer.model.TweetWithCandidateGenerationInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.param.InterestedInParams import com.twitter.cr_mixer.param.SimClustersANNParams import com.twitter.cr_mixer.similarity_engine.EngineQuery import com.twitter.cr_mixer.similarity_engine.SimClustersANNSimilarityEngine import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.base.CandidateSource import com.twitter.frigate.common.util.StatsUtil import com.twitter.simclusters_v2.common.ModelVersions import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.timelines.configapi import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton import javax.inject.Named import com.twitter.cr_mixer.model.ModuleNames /** * This store looks for similar tweets for a given UserId that generates UserInterestedIn * from SimClustersANN. It will be a standalone CandidateGeneration class moving forward. * * After the abstraction improvement (apply SimilarityEngine trait) * these CG will be subjected to change. */ @Singleton case class SimClustersInterestedInCandidateGeneration @Inject() ( @Named(ModuleNames.SimClustersANNSimilarityEngine) simClustersANNSimilarityEngine: StandardSimilarityEngine[ SimClustersANNSimilarityEngine.Query, TweetWithScore ], statsReceiver: StatsReceiver) extends CandidateSource[ SimClustersInterestedInCandidateGeneration.Query, Seq[TweetWithCandidateGenerationInfo] ] { override def name: String = this.getClass.getSimpleName private val stats = statsReceiver.scope(name) private val fetchCandidatesStat = stats.scope("fetchCandidates") override def get( query: SimClustersInterestedInCandidateGeneration.Query ): Future[Option[Seq[Seq[TweetWithCandidateGenerationInfo]]]] = { query.internalId match { case _: InternalId.UserId => StatsUtil.trackOptionItemsStats(fetchCandidatesStat) { // UserInterestedIn Queries val userInterestedInCandidateResultFut = if (query.enableUserInterestedIn && query.enableProdSimClustersANNSimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.interestedInSimClustersANNQuery, query.simClustersInterestedInMinScore) else Future.None val userInterestedInExperimentalSANNCandidateResultFut = if (query.enableUserInterestedIn && query.enableExperimentalSimClustersANNSimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.interestedInExperimentalSimClustersANNQuery, query.simClustersInterestedInMinScore) else Future.None val userInterestedInSANN1CandidateResultFut = if (query.enableUserInterestedIn && query.enableSimClustersANN1SimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.interestedInSimClustersANN1Query, query.simClustersInterestedInMinScore) else Future.None val userInterestedInSANN2CandidateResultFut = if (query.enableUserInterestedIn && query.enableSimClustersANN2SimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.interestedInSimClustersANN2Query, query.simClustersInterestedInMinScore) else Future.None val userInterestedInSANN3CandidateResultFut = if (query.enableUserInterestedIn && query.enableSimClustersANN3SimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.interestedInSimClustersANN3Query, query.simClustersInterestedInMinScore) else Future.None val userInterestedInSANN5CandidateResultFut = if (query.enableUserInterestedIn && query.enableSimClustersANN5SimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.interestedInSimClustersANN5Query, query.simClustersInterestedInMinScore) else Future.None val userInterestedInSANN4CandidateResultFut = if (query.enableUserInterestedIn && query.enableSimClustersANN4SimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.interestedInSimClustersANN4Query, query.simClustersInterestedInMinScore) else Future.None // UserNextInterestedIn Queries val userNextInterestedInCandidateResultFut = if (query.enableUserNextInterestedIn && query.enableProdSimClustersANNSimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.nextInterestedInSimClustersANNQuery, query.simClustersInterestedInMinScore) else Future.None val userNextInterestedInExperimentalSANNCandidateResultFut = if (query.enableUserNextInterestedIn && query.enableExperimentalSimClustersANNSimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.nextInterestedInExperimentalSimClustersANNQuery, query.simClustersInterestedInMinScore) else Future.None val userNextInterestedInSANN1CandidateResultFut = if (query.enableUserNextInterestedIn && query.enableSimClustersANN1SimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.nextInterestedInSimClustersANN1Query, query.simClustersInterestedInMinScore) else Future.None val userNextInterestedInSANN2CandidateResultFut = if (query.enableUserNextInterestedIn && query.enableSimClustersANN2SimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.nextInterestedInSimClustersANN2Query, query.simClustersInterestedInMinScore) else Future.None val userNextInterestedInSANN3CandidateResultFut = if (query.enableUserNextInterestedIn && query.enableSimClustersANN3SimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.nextInterestedInSimClustersANN3Query, query.simClustersInterestedInMinScore) else Future.None val userNextInterestedInSANN5CandidateResultFut = if (query.enableUserNextInterestedIn && query.enableSimClustersANN5SimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.nextInterestedInSimClustersANN5Query, query.simClustersInterestedInMinScore) else Future.None val userNextInterestedInSANN4CandidateResultFut = if (query.enableUserNextInterestedIn && query.enableSimClustersANN4SimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.nextInterestedInSimClustersANN4Query, query.simClustersInterestedInMinScore) else Future.None // AddressBookInterestedIn Queries val userAddressBookInterestedInCandidateResultFut = if (query.enableAddressBookNextInterestedIn && query.enableProdSimClustersANNSimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.addressbookInterestedInSimClustersANNQuery, query.simClustersInterestedInMinScore) else Future.None val userAddressBookExperimentalSANNCandidateResultFut = if (query.enableAddressBookNextInterestedIn && query.enableExperimentalSimClustersANNSimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.addressbookInterestedInExperimentalSimClustersANNQuery, query.simClustersInterestedInMinScore) else Future.None val userAddressBookSANN1CandidateResultFut = if (query.enableAddressBookNextInterestedIn && query.enableSimClustersANN1SimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.addressbookInterestedInSimClustersANN1Query, query.simClustersInterestedInMinScore) else Future.None val userAddressBookSANN2CandidateResultFut = if (query.enableAddressBookNextInterestedIn && query.enableSimClustersANN2SimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.addressbookInterestedInSimClustersANN2Query, query.simClustersInterestedInMinScore) else Future.None val userAddressBookSANN3CandidateResultFut = if (query.enableAddressBookNextInterestedIn && query.enableSimClustersANN3SimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.addressbookInterestedInSimClustersANN3Query, query.simClustersInterestedInMinScore) else Future.None val userAddressBookSANN5CandidateResultFut = if (query.enableAddressBookNextInterestedIn && query.enableSimClustersANN5SimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.addressbookInterestedInSimClustersANN5Query, query.simClustersInterestedInMinScore) else Future.None val userAddressBookSANN4CandidateResultFut = if (query.enableAddressBookNextInterestedIn && query.enableSimClustersANN4SimilarityEngine) getInterestedInCandidateResult( simClustersANNSimilarityEngine, query.addressbookInterestedInSimClustersANN4Query, query.simClustersInterestedInMinScore) else Future.None Future .collect( Seq( userInterestedInCandidateResultFut, userNextInterestedInCandidateResultFut, userAddressBookInterestedInCandidateResultFut, userInterestedInExperimentalSANNCandidateResultFut, userNextInterestedInExperimentalSANNCandidateResultFut, userAddressBookExperimentalSANNCandidateResultFut, userInterestedInSANN1CandidateResultFut, userNextInterestedInSANN1CandidateResultFut, userAddressBookSANN1CandidateResultFut, userInterestedInSANN2CandidateResultFut, userNextInterestedInSANN2CandidateResultFut, userAddressBookSANN2CandidateResultFut, userInterestedInSANN3CandidateResultFut, userNextInterestedInSANN3CandidateResultFut, userAddressBookSANN3CandidateResultFut, userInterestedInSANN5CandidateResultFut, userNextInterestedInSANN5CandidateResultFut, userAddressBookSANN5CandidateResultFut, userInterestedInSANN4CandidateResultFut, userNextInterestedInSANN4CandidateResultFut, userAddressBookSANN4CandidateResultFut ) ).map { candidateResults => Some( candidateResults.map(candidateResult => candidateResult.getOrElse(Seq.empty)) ) } } case _ => stats.counter("sourceId_is_not_userId_cnt").incr() Future.None } } private def simClustersCandidateMinScoreFilter( simClustersAnnCandidates: Seq[TweetWithScore], simClustersInterestedInMinScore: Double, simClustersANNConfigId: String ): Seq[TweetWithScore] = { val filteredCandidates = simClustersAnnCandidates .filter { candidate => candidate.score > simClustersInterestedInMinScore } stats.stat(simClustersANNConfigId, "simClustersAnnCandidates_size").add(filteredCandidates.size) stats.counter(simClustersANNConfigId, "simClustersAnnRequests").incr() if (filteredCandidates.isEmpty) stats.counter(simClustersANNConfigId, "emptyFilteredSimClustersAnnCandidates").incr() filteredCandidates.map { candidate => TweetWithScore(candidate.tweetId, candidate.score) } } private def getInterestedInCandidateResult( simClustersANNSimilarityEngine: StandardSimilarityEngine[ SimClustersANNSimilarityEngine.Query, TweetWithScore ], simClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], simClustersInterestedInMinScore: Double, ): Future[Option[Seq[TweetWithCandidateGenerationInfo]]] = { val interestedInCandidatesFut = simClustersANNSimilarityEngine.getCandidates(simClustersANNQuery) val interestedInCandidateResultFut = interestedInCandidatesFut.map { interestedInCandidates => stats.stat("candidateSize").add(interestedInCandidates.size) val embeddingCandidatesStat = stats.scope( simClustersANNQuery.storeQuery.simClustersANNQuery.sourceEmbeddingId.embeddingType.name) embeddingCandidatesStat.stat("candidateSize").add(interestedInCandidates.size) if (interestedInCandidates.isEmpty) { embeddingCandidatesStat.counter("empty_results").incr() } embeddingCandidatesStat.counter("requests").incr() val filteredTweets = simClustersCandidateMinScoreFilter( interestedInCandidates.toSeq.flatten, simClustersInterestedInMinScore, simClustersANNQuery.storeQuery.simClustersANNConfigId) val interestedInTweetsWithCGInfo = filteredTweets.map { tweetWithScore => TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( None, SimClustersANNSimilarityEngine .toSimilarityEngineInfo(simClustersANNQuery, tweetWithScore.score), Seq.empty // SANN is an atomic SE, and hence it has no contributing SEs ) ) } val interestedInResults = if (interestedInTweetsWithCGInfo.nonEmpty) { Some(interestedInTweetsWithCGInfo) } else None interestedInResults } interestedInCandidateResultFut } } object SimClustersInterestedInCandidateGeneration { case class Query( internalId: InternalId, enableUserInterestedIn: Boolean, enableUserNextInterestedIn: Boolean, enableAddressBookNextInterestedIn: Boolean, enableProdSimClustersANNSimilarityEngine: Boolean, enableExperimentalSimClustersANNSimilarityEngine: Boolean, enableSimClustersANN1SimilarityEngine: Boolean, enableSimClustersANN2SimilarityEngine: Boolean, enableSimClustersANN3SimilarityEngine: Boolean, enableSimClustersANN5SimilarityEngine: Boolean, enableSimClustersANN4SimilarityEngine: Boolean, simClustersInterestedInMinScore: Double, simClustersNextInterestedInMinScore: Double, simClustersAddressBookInterestedInMinScore: Double, interestedInSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], nextInterestedInSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], addressbookInterestedInSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], interestedInExperimentalSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], nextInterestedInExperimentalSimClustersANNQuery: EngineQuery[ SimClustersANNSimilarityEngine.Query ], addressbookInterestedInExperimentalSimClustersANNQuery: EngineQuery[ SimClustersANNSimilarityEngine.Query ], interestedInSimClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query], nextInterestedInSimClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query], addressbookInterestedInSimClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query], interestedInSimClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query], nextInterestedInSimClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query], addressbookInterestedInSimClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query], interestedInSimClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query], nextInterestedInSimClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query], addressbookInterestedInSimClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query], interestedInSimClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query], nextInterestedInSimClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query], addressbookInterestedInSimClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query], interestedInSimClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query], nextInterestedInSimClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query], addressbookInterestedInSimClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query], ) def fromParams( internalId: InternalId, params: configapi.Params, ): Query = { // SimClusters common configs val simClustersModelVersion = ModelVersions.Enum.enumToSimClustersModelVersionMap(params(GlobalParams.ModelVersionParam)) val simClustersANNConfigId = params(SimClustersANNParams.SimClustersANNConfigId) val experimentalSimClustersANNConfigId = params( SimClustersANNParams.ExperimentalSimClustersANNConfigId) val simClustersANN1ConfigId = params(SimClustersANNParams.SimClustersANN1ConfigId) val simClustersANN2ConfigId = params(SimClustersANNParams.SimClustersANN2ConfigId) val simClustersANN3ConfigId = params(SimClustersANNParams.SimClustersANN3ConfigId) val simClustersANN5ConfigId = params(SimClustersANNParams.SimClustersANN5ConfigId) val simClustersANN4ConfigId = params(SimClustersANNParams.SimClustersANN4ConfigId) val simClustersInterestedInMinScore = params(InterestedInParams.MinScoreParam) val simClustersNextInterestedInMinScore = params( InterestedInParams.MinScoreSequentialModelParam) val simClustersAddressBookInterestedInMinScore = params( InterestedInParams.MinScoreAddressBookParam) // InterestedIn embeddings parameters val interestedInEmbedding = params(InterestedInParams.InterestedInEmbeddingIdParam) val nextInterestedInEmbedding = params(InterestedInParams.NextInterestedInEmbeddingIdParam) val addressbookInterestedInEmbedding = params( InterestedInParams.AddressBookInterestedInEmbeddingIdParam) // Prod SimClustersANN Query val interestedInSimClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( internalId, interestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANNConfigId, params) val nextInterestedInSimClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( internalId, nextInterestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANNConfigId, params) val addressbookInterestedInSimClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( internalId, addressbookInterestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANNConfigId, params) // Experimental SANN cluster Query val interestedInExperimentalSimClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( internalId, interestedInEmbedding.embeddingType, simClustersModelVersion, experimentalSimClustersANNConfigId, params) val nextInterestedInExperimentalSimClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( internalId, nextInterestedInEmbedding.embeddingType, simClustersModelVersion, experimentalSimClustersANNConfigId, params) val addressbookInterestedInExperimentalSimClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( internalId, addressbookInterestedInEmbedding.embeddingType, simClustersModelVersion, experimentalSimClustersANNConfigId, params) // SimClusters ANN cluster 1 Query val interestedInSimClustersANN1Query = SimClustersANNSimilarityEngine.fromParams( internalId, interestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANN1ConfigId, params) val nextInterestedInSimClustersANN1Query = SimClustersANNSimilarityEngine.fromParams( internalId, nextInterestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANN1ConfigId, params) val addressbookInterestedInSimClustersANN1Query = SimClustersANNSimilarityEngine.fromParams( internalId, addressbookInterestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANN1ConfigId, params) // SimClusters ANN cluster 2 Query val interestedInSimClustersANN2Query = SimClustersANNSimilarityEngine.fromParams( internalId, interestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANN2ConfigId, params) val nextInterestedInSimClustersANN2Query = SimClustersANNSimilarityEngine.fromParams( internalId, nextInterestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANN2ConfigId, params) val addressbookInterestedInSimClustersANN2Query = SimClustersANNSimilarityEngine.fromParams( internalId, addressbookInterestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANN2ConfigId, params) // SimClusters ANN cluster 3 Query val interestedInSimClustersANN3Query = SimClustersANNSimilarityEngine.fromParams( internalId, interestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANN3ConfigId, params) val nextInterestedInSimClustersANN3Query = SimClustersANNSimilarityEngine.fromParams( internalId, nextInterestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANN3ConfigId, params) val addressbookInterestedInSimClustersANN3Query = SimClustersANNSimilarityEngine.fromParams( internalId, addressbookInterestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANN3ConfigId, params) // SimClusters ANN cluster 5 Query val interestedInSimClustersANN5Query = SimClustersANNSimilarityEngine.fromParams( internalId, interestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANN5ConfigId, params) // SimClusters ANN cluster 4 Query val interestedInSimClustersANN4Query = SimClustersANNSimilarityEngine.fromParams( internalId, interestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANN4ConfigId, params) val nextInterestedInSimClustersANN5Query = SimClustersANNSimilarityEngine.fromParams( internalId, nextInterestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANN5ConfigId, params) val nextInterestedInSimClustersANN4Query = SimClustersANNSimilarityEngine.fromParams( internalId, nextInterestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANN4ConfigId, params) val addressbookInterestedInSimClustersANN5Query = SimClustersANNSimilarityEngine.fromParams( internalId, addressbookInterestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANN5ConfigId, params) val addressbookInterestedInSimClustersANN4Query = SimClustersANNSimilarityEngine.fromParams( internalId, addressbookInterestedInEmbedding.embeddingType, simClustersModelVersion, simClustersANN4ConfigId, params) Query( internalId = internalId, enableUserInterestedIn = params(InterestedInParams.EnableSourceParam), enableUserNextInterestedIn = params(InterestedInParams.EnableSourceSequentialModelParam), enableAddressBookNextInterestedIn = params(InterestedInParams.EnableSourceAddressBookParam), enableProdSimClustersANNSimilarityEngine = params(InterestedInParams.EnableProdSimClustersANNParam), enableExperimentalSimClustersANNSimilarityEngine = params(InterestedInParams.EnableExperimentalSimClustersANNParam), enableSimClustersANN1SimilarityEngine = params(InterestedInParams.EnableSimClustersANN1Param), enableSimClustersANN2SimilarityEngine = params(InterestedInParams.EnableSimClustersANN2Param), enableSimClustersANN3SimilarityEngine = params(InterestedInParams.EnableSimClustersANN3Param), enableSimClustersANN5SimilarityEngine = params(InterestedInParams.EnableSimClustersANN5Param), enableSimClustersANN4SimilarityEngine = params(InterestedInParams.EnableSimClustersANN4Param), simClustersInterestedInMinScore = simClustersInterestedInMinScore, simClustersNextInterestedInMinScore = simClustersNextInterestedInMinScore, simClustersAddressBookInterestedInMinScore = simClustersAddressBookInterestedInMinScore, interestedInSimClustersANNQuery = interestedInSimClustersANNQuery, nextInterestedInSimClustersANNQuery = nextInterestedInSimClustersANNQuery, addressbookInterestedInSimClustersANNQuery = addressbookInterestedInSimClustersANNQuery, interestedInExperimentalSimClustersANNQuery = interestedInExperimentalSimClustersANNQuery, nextInterestedInExperimentalSimClustersANNQuery = nextInterestedInExperimentalSimClustersANNQuery, addressbookInterestedInExperimentalSimClustersANNQuery = addressbookInterestedInExperimentalSimClustersANNQuery, interestedInSimClustersANN1Query = interestedInSimClustersANN1Query, nextInterestedInSimClustersANN1Query = nextInterestedInSimClustersANN1Query, addressbookInterestedInSimClustersANN1Query = addressbookInterestedInSimClustersANN1Query, interestedInSimClustersANN2Query = interestedInSimClustersANN2Query, nextInterestedInSimClustersANN2Query = nextInterestedInSimClustersANN2Query, addressbookInterestedInSimClustersANN2Query = addressbookInterestedInSimClustersANN2Query, interestedInSimClustersANN3Query = interestedInSimClustersANN3Query, nextInterestedInSimClustersANN3Query = nextInterestedInSimClustersANN3Query, addressbookInterestedInSimClustersANN3Query = addressbookInterestedInSimClustersANN3Query, interestedInSimClustersANN5Query = interestedInSimClustersANN5Query, nextInterestedInSimClustersANN5Query = nextInterestedInSimClustersANN5Query, addressbookInterestedInSimClustersANN5Query = addressbookInterestedInSimClustersANN5Query, interestedInSimClustersANN4Query = interestedInSimClustersANN4Query, nextInterestedInSimClustersANN4Query = nextInterestedInSimClustersANN4Query, addressbookInterestedInSimClustersANN4Query = addressbookInterestedInSimClustersANN4Query, ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation/TopicTweetCandidateGenerator.scala ================================================ package com.twitter.cr_mixer.candidate_generation import com.twitter.contentrecommender.thriftscala.TweetInfo import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.CandidateGenerationInfo import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.TopicTweetCandidateGeneratorQuery import com.twitter.cr_mixer.model.TopicTweetWithScore import com.twitter.cr_mixer.param.TopicTweetParams import com.twitter.cr_mixer.similarity_engine.CertoTopicTweetSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SkitHighPrecisionTopicTweetSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SkitTopicTweetSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.cr_mixer.thriftscala.TopicTweet import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.util.DefaultTimer import com.twitter.frigate.common.util.StatsUtil import com.twitter.servo.util.MemoizingStatsReceiver import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.thriftscala.TopicId import com.twitter.snowflake.id.SnowflakeId import com.twitter.storehaus.ReadableStore import com.twitter.util.Duration import com.twitter.util.Future import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton /** * Formerly CrTopic in legacy Content Recommender. This generator finds top Tweets per Topic. */ @Singleton class TopicTweetCandidateGenerator @Inject() ( certoTopicTweetSimilarityEngine: CertoTopicTweetSimilarityEngine, skitTopicTweetSimilarityEngine: SkitTopicTweetSimilarityEngine, skitHighPrecisionTopicTweetSimilarityEngine: SkitHighPrecisionTopicTweetSimilarityEngine, tweetInfoStore: ReadableStore[TweetId, TweetInfo], timeoutConfig: TimeoutConfig, globalStats: StatsReceiver) { private val timer = DefaultTimer private val stats: StatsReceiver = globalStats.scope(this.getClass.getCanonicalName) private val fetchCandidatesStats = stats.scope("fetchCandidates") private val filterCandidatesStats = stats.scope("filterCandidates") private val tweetyPieFilteredStats = filterCandidatesStats.stat("tweetypie_filtered") private val memoizedStatsReceiver = new MemoizingStatsReceiver(stats) def get( query: TopicTweetCandidateGeneratorQuery ): Future[Map[Long, Seq[TopicTweet]]] = { val maxTweetAge = query.params(TopicTweetParams.MaxTweetAge) val product = query.product val allStats = memoizedStatsReceiver.scope("all") val perProductStats = memoizedStatsReceiver.scope("perProduct", product.name) StatsUtil.trackMapValueStats(allStats) { StatsUtil.trackMapValueStats(perProductStats) { val result = for { retrievedTweets <- fetchCandidates(query) initialTweetCandidates <- convertToInitialCandidates(retrievedTweets) filteredTweetCandidates <- filterCandidates( initialTweetCandidates, maxTweetAge, query.isVideoOnly, query.impressedTweetList) rankedTweetCandidates = rankCandidates(filteredTweetCandidates) hydratedTweetCandidates = hydrateCandidates(rankedTweetCandidates) } yield { hydratedTweetCandidates.map { case (topicId, topicTweets) => val topKTweets = topicTweets.take(query.maxNumResults) topicId -> topKTweets } } result.raiseWithin(timeoutConfig.topicTweetEndpointTimeout)(timer) } } } private def fetchCandidates( query: TopicTweetCandidateGeneratorQuery ): Future[Map[TopicId, Option[Seq[TopicTweetWithScore]]]] = { Future.collect { query.topicIds.map { topicId => topicId -> StatsUtil.trackOptionStats(fetchCandidatesStats) { Future .join( certoTopicTweetSimilarityEngine.get(CertoTopicTweetSimilarityEngine .fromParams(topicId, query.isVideoOnly, query.params)), skitTopicTweetSimilarityEngine .get(SkitTopicTweetSimilarityEngine .fromParams(topicId, query.isVideoOnly, query.params)), skitHighPrecisionTopicTweetSimilarityEngine .get(SkitHighPrecisionTopicTweetSimilarityEngine .fromParams(topicId, query.isVideoOnly, query.params)) ).map { case (certoTopicTweets, skitTfgTopicTweets, skitHighPrecisionTopicTweets) => val uniqueCandidates = (certoTopicTweets.getOrElse(Nil) ++ skitTfgTopicTweets.getOrElse(Nil) ++ skitHighPrecisionTopicTweets.getOrElse(Nil)) .groupBy(_.tweetId).map { case (_, dupCandidates) => dupCandidates.head }.toSeq Some(uniqueCandidates) } } }.toMap } } private def convertToInitialCandidates( candidatesMap: Map[TopicId, Option[Seq[TopicTweetWithScore]]] ): Future[Map[TopicId, Seq[InitialCandidate]]] = { val initialCandidates = candidatesMap.map { case (topicId, candidatesOpt) => val candidates = candidatesOpt.getOrElse(Nil) val tweetIds = candidates.map(_.tweetId).toSet val numTweetsPreFilter = tweetIds.size Future.collect(tweetInfoStore.multiGet(tweetIds)).map { tweetInfos => /** * * If tweetInfo does not exist, we will filter out this tweet candidate. */ val tweetyPieFilteredInitialCandidates = candidates.collect { case candidate if tweetInfos.getOrElse(candidate.tweetId, None).isDefined => val tweetInfo = tweetInfos(candidate.tweetId) .getOrElse(throw new IllegalStateException("Check previous line's condition")) InitialCandidate( tweetId = candidate.tweetId, tweetInfo = tweetInfo, CandidateGenerationInfo( None, SimilarityEngineInfo( similarityEngineType = candidate.similarityEngineType, modelId = None, score = Some(candidate.score)), Seq.empty ) ) } val numTweetsPostFilter = tweetyPieFilteredInitialCandidates.size tweetyPieFilteredStats.add(numTweetsPreFilter - numTweetsPostFilter) topicId -> tweetyPieFilteredInitialCandidates } } Future.collect(initialCandidates.toSeq).map(_.toMap) } private def filterCandidates( topicTweetMap: Map[TopicId, Seq[InitialCandidate]], maxTweetAge: Duration, isVideoOnly: Boolean, excludeTweetIds: Set[TweetId] ): Future[Map[TopicId, Seq[InitialCandidate]]] = { val earliestTweetId = SnowflakeId.firstIdFor(Time.now - maxTweetAge) val filteredResults = topicTweetMap.map { case (topicId, tweetsWithScore) => topicId -> StatsUtil.trackItemsStats(filterCandidatesStats) { val timeFilteredTweets = tweetsWithScore.filter { tweetWithScore => tweetWithScore.tweetId >= earliestTweetId && !excludeTweetIds.contains( tweetWithScore.tweetId) } filterCandidatesStats .stat("exclude_and_time_filtered").add(tweetsWithScore.size - timeFilteredTweets.size) val tweetNudityFilteredTweets = timeFilteredTweets.collect { case tweet if tweet.tweetInfo.isPassTweetMediaNudityTag.contains(true) => tweet } filterCandidatesStats .stat("tweet_nudity_filtered").add( timeFilteredTweets.size - tweetNudityFilteredTweets.size) val userNudityFilteredTweets = tweetNudityFilteredTweets.collect { case tweet if tweet.tweetInfo.isPassUserNudityRateStrict.contains(true) => tweet } filterCandidatesStats .stat("user_nudity_filtered").add( tweetNudityFilteredTweets.size - userNudityFilteredTweets.size) val videoFilteredTweets = { if (isVideoOnly) { userNudityFilteredTweets.collect { case tweet if tweet.tweetInfo.hasVideo.contains(true) => tweet } } else { userNudityFilteredTweets } } Future.value(videoFilteredTweets) } } Future.collect(filteredResults) } private def rankCandidates( tweetCandidatesMap: Map[TopicId, Seq[InitialCandidate]] ): Map[TopicId, Seq[InitialCandidate]] = { tweetCandidatesMap.mapValues { tweetCandidates => tweetCandidates.sortBy { candidate => -candidate.tweetInfo.favCount } } } private def hydrateCandidates( topicCandidatesMap: Map[TopicId, Seq[InitialCandidate]] ): Map[Long, Seq[TopicTweet]] = { topicCandidatesMap.map { case (topicId, tweetsWithScore) => topicId.entityId -> tweetsWithScore.map { tweetWithScore => val similarityEngineType: SimilarityEngineType = tweetWithScore.candidateGenerationInfo.similarityEngineInfo.similarityEngineType TopicTweet( tweetId = tweetWithScore.tweetId, score = tweetWithScore.getSimilarityScore, similarityEngineType = similarityEngineType ) } } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation/UtegTweetCandidateGenerator.scala ================================================ package com.twitter.cr_mixer.candidate_generation import com.twitter.contentrecommender.thriftscala.TweetInfo import com.twitter.cr_mixer.logging.UtegTweetScribeLogger import com.twitter.cr_mixer.filter.UtegFilterRunner import com.twitter.cr_mixer.model.CandidateGenerationInfo import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.RankedCandidate import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.TweetWithScoreAndSocialProof import com.twitter.cr_mixer.model.UtegTweetCandidateGeneratorQuery import com.twitter.cr_mixer.similarity_engine.UserTweetEntityGraphSimilarityEngine import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.source_signal.RealGraphInSourceGraphFetcher import com.twitter.cr_mixer.source_signal.SourceFetcher.FetcherQuery import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.common.UserId import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class UtegTweetCandidateGenerator @Inject() ( @Named(ModuleNames.UserTweetEntityGraphSimilarityEngine) userTweetEntityGraphSimilarityEngine: StandardSimilarityEngine[ UserTweetEntityGraphSimilarityEngine.Query, TweetWithScoreAndSocialProof ], utegTweetScribeLogger: UtegTweetScribeLogger, tweetInfoStore: ReadableStore[TweetId, TweetInfo], realGraphInSourceGraphFetcher: RealGraphInSourceGraphFetcher, utegFilterRunner: UtegFilterRunner, globalStats: StatsReceiver) { private val stats: StatsReceiver = globalStats.scope(this.getClass.getCanonicalName) private val fetchSeedsStats = stats.scope("fetchSeeds") private val fetchCandidatesStats = stats.scope("fetchCandidates") private val utegFilterStats = stats.scope("utegFilter") private val rankStats = stats.scope("rank") def get( query: UtegTweetCandidateGeneratorQuery ): Future[Seq[TweetWithScoreAndSocialProof]] = { val allStats = stats.scope("all") val perProductStats = stats.scope("perProduct", query.product.toString) StatsUtil.trackItemsStats(allStats) { StatsUtil.trackItemsStats(perProductStats) { /** * The candidate we return in the end needs a social proof field, which isn't * supported by the any existing Candidate type, so we created TweetWithScoreAndSocialProof * instead. * * However, filters and light ranker expect Candidate-typed param to work. In order to minimise the * changes to them, we are doing conversions from/to TweetWithScoreAndSocialProof to/from Candidate * in this method. */ for { realGraphSeeds <- StatsUtil.trackItemMapStats(fetchSeedsStats) { fetchSeeds(query) } initialTweets <- StatsUtil.trackItemsStats(fetchCandidatesStats) { fetchCandidates(query, realGraphSeeds) } initialCandidates <- convertToInitialCandidates(initialTweets) filteredCandidates <- StatsUtil.trackItemsStats(utegFilterStats) { utegFilter(query, initialCandidates) } rankedCandidates <- StatsUtil.trackItemsStats(rankStats) { rankCandidates(query, filteredCandidates) } } yield { val topTweets = rankedCandidates.take(query.maxNumResults) convertToTweets(topTweets, initialTweets.map(tweet => tweet.tweetId -> tweet).toMap) } } } } private def utegFilter( query: UtegTweetCandidateGeneratorQuery, candidates: Seq[InitialCandidate] ): Future[Seq[InitialCandidate]] = { utegFilterRunner.runSequentialFilters(query, Seq(candidates)).map(_.flatten) } private def fetchSeeds( query: UtegTweetCandidateGeneratorQuery ): Future[Map[UserId, Double]] = { realGraphInSourceGraphFetcher .get(FetcherQuery(query.userId, query.product, query.userState, query.params)) .map(_.map(_.seedWithScores).getOrElse(Map.empty)) } private[candidate_generation] def rankCandidates( query: UtegTweetCandidateGeneratorQuery, filteredCandidates: Seq[InitialCandidate], ): Future[Seq[RankedCandidate]] = { val blendedCandidates = filteredCandidates.map(candidate => candidate.toBlendedCandidate(Seq(candidate.candidateGenerationInfo))) Future( blendedCandidates.map { candidate => val score = candidate.getSimilarityScore candidate.toRankedCandidate(score) } ) } def fetchCandidates( query: UtegTweetCandidateGeneratorQuery, realGraphSeeds: Map[UserId, Double], ): Future[Seq[TweetWithScoreAndSocialProof]] = { val engineQuery = UserTweetEntityGraphSimilarityEngine.fromParams( query.userId, realGraphSeeds, Some(query.impressedTweetList.toSeq), query.params ) utegTweetScribeLogger.scribeInitialCandidates( query, userTweetEntityGraphSimilarityEngine.getCandidates(engineQuery).map(_.toSeq.flatten) ) } private[candidate_generation] def convertToInitialCandidates( candidates: Seq[TweetWithScoreAndSocialProof], ): Future[Seq[InitialCandidate]] = { val tweetIds = candidates.map(_.tweetId).toSet Future.collect(tweetInfoStore.multiGet(tweetIds)).map { tweetInfos => /** * * If tweetInfo does not exist, we will filter out this tweet candidate. */ candidates.collect { case candidate if tweetInfos.getOrElse(candidate.tweetId, None).isDefined => val tweetInfo = tweetInfos(candidate.tweetId) .getOrElse(throw new IllegalStateException("Check previous line's condition")) InitialCandidate( tweetId = candidate.tweetId, tweetInfo = tweetInfo, CandidateGenerationInfo( None, SimilarityEngineInfo( similarityEngineType = SimilarityEngineType.Uteg, modelId = None, score = Some(candidate.score)), Seq.empty ) ) } } } private[candidate_generation] def convertToTweets( candidates: Seq[RankedCandidate], tweetMap: Map[TweetId, TweetWithScoreAndSocialProof] ): Seq[TweetWithScoreAndSocialProof] = { candidates.map { candidate => tweetMap .get(candidate.tweetId).map { tweet => TweetWithScoreAndSocialProof( tweet.tweetId, candidate.predictionScore, tweet.socialProofByType ) // The exception should never be thrown }.getOrElse(throw new Exception("Cannot find ranked candidate in original UTEG tweets")) } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/config/BUILD ================================================ scala_library( sources = ["*.scala"], tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/javax/inject:javax.inject", "configapi/configapi-core", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/exception", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", "finatra/inject/inject-core/src/main/scala", "simclusters-ann/thrift/src/main/thrift:thrift-scala", "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/config/SimClustersANNConfig.scala ================================================ package com.twitter.cr_mixer.config import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.exception.InvalidSANNConfigException import com.twitter.simclusters_v2.thriftscala.EmbeddingType import com.twitter.simclustersann.thriftscala.ScoringAlgorithm import com.twitter.simclustersann.thriftscala.{SimClustersANNConfig => ThriftSimClustersANNConfig} import com.twitter.util.Duration case class SimClustersANNConfig( maxNumResults: Int, minScore: Double, candidateEmbeddingType: EmbeddingType, maxTopTweetsPerCluster: Int, maxScanClusters: Int, maxTweetCandidateAge: Duration, minTweetCandidateAge: Duration, annAlgorithm: ScoringAlgorithm) { val toSANNConfigThrift: ThriftSimClustersANNConfig = ThriftSimClustersANNConfig( maxNumResults = maxNumResults, minScore = minScore, candidateEmbeddingType = candidateEmbeddingType, maxTopTweetsPerCluster = maxTopTweetsPerCluster, maxScanClusters = maxScanClusters, maxTweetCandidateAgeHours = maxTweetCandidateAge.inHours, minTweetCandidateAgeHours = minTweetCandidateAge.inHours, annAlgorithm = annAlgorithm, ) } object SimClustersANNConfig { final val DefaultConfig = SimClustersANNConfig( maxNumResults = 200, minScore = 0.0, candidateEmbeddingType = EmbeddingType.LogFavBasedTweet, maxTopTweetsPerCluster = 800, maxScanClusters = 50, maxTweetCandidateAge = 24.hours, minTweetCandidateAge = 0.hours, annAlgorithm = ScoringAlgorithm.CosineSimilarity, ) /* SimClustersANNConfigId: String Format: Prod - “EmbeddingType_ModelVersion_Default” Format: Experiment - “EmbeddingType_ModelVersion_Date_Two-Digit-Serial-Number”. Date : YYYYMMDD */ private val FavBasedProducer_Model20m145k2020_Default = DefaultConfig.copy() // Chunnan's exp on maxTweetCandidateAgeDays 2 private val FavBasedProducer_Model20m145k2020_20220617_06 = FavBasedProducer_Model20m145k2020_Default.copy( maxTweetCandidateAge = 48.hours, ) // Experimental SANN config private val FavBasedProducer_Model20m145k2020_20220801 = FavBasedProducer_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.VideoPlayBack50LogFavBasedTweet, ) // SANN-1 config private val FavBasedProducer_Model20m145k2020_20220810 = FavBasedProducer_Model20m145k2020_Default.copy( maxNumResults = 100, candidateEmbeddingType = EmbeddingType.LogFavBasedAdsTweet, maxTweetCandidateAge = 175200.hours, maxTopTweetsPerCluster = 1600 ) // SANN-2 config private val FavBasedProducer_Model20m145k2020_20220818 = FavBasedProducer_Model20m145k2020_Default.copy( maxNumResults = 100, candidateEmbeddingType = EmbeddingType.LogFavClickBasedAdsTweet, maxTweetCandidateAge = 175200.hours, maxTopTweetsPerCluster = 1600 ) // SANN-3 config private val FavBasedProducer_Model20m145k2020_20220819 = FavBasedProducer_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.PushOpenLogFavBasedTweet, ) // SANN-5 config private val FavBasedProducer_Model20m145k2020_20221221 = FavBasedProducer_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.LogFavBasedRealTimeTweet, maxTweetCandidateAge = 1.hours ) // SANN-4 config private val FavBasedProducer_Model20m145k2020_20221220 = FavBasedProducer_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.LogFavBasedEvergreenTweet, maxTweetCandidateAge = 48.hours ) private val LogFavLongestL2EmbeddingTweet_Model20m145k2020_Default = DefaultConfig.copy() // Chunnan's exp on maxTweetCandidateAgeDays 2 private val LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220617_06 = LogFavLongestL2EmbeddingTweet_Model20m145k2020_Default.copy( maxTweetCandidateAge = 48.hours, ) // Experimental SANN config private val LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220801 = LogFavLongestL2EmbeddingTweet_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.VideoPlayBack50LogFavBasedTweet, ) // SANN-1 config private val LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220810 = LogFavLongestL2EmbeddingTweet_Model20m145k2020_Default.copy( maxNumResults = 100, candidateEmbeddingType = EmbeddingType.LogFavBasedAdsTweet, maxTweetCandidateAge = 175200.hours, maxTopTweetsPerCluster = 1600 ) // SANN-2 config private val LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220818 = LogFavLongestL2EmbeddingTweet_Model20m145k2020_Default.copy( maxNumResults = 100, candidateEmbeddingType = EmbeddingType.LogFavClickBasedAdsTweet, maxTweetCandidateAge = 175200.hours, maxTopTweetsPerCluster = 1600 ) // SANN-3 config private val LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220819 = LogFavLongestL2EmbeddingTweet_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.PushOpenLogFavBasedTweet, ) // SANN-5 config private val LogFavLongestL2EmbeddingTweet_Model20m145k2020_20221221 = LogFavLongestL2EmbeddingTweet_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.LogFavBasedRealTimeTweet, maxTweetCandidateAge = 1.hours ) // SANN-4 config private val LogFavLongestL2EmbeddingTweet_Model20m145k2020_20221220 = LogFavLongestL2EmbeddingTweet_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.LogFavBasedEvergreenTweet, maxTweetCandidateAge = 48.hours ) private val UnfilteredUserInterestedIn_Model20m145k2020_Default = DefaultConfig.copy() // Chunnan's exp on maxTweetCandidateAgeDays 2 private val UnfilteredUserInterestedIn_Model20m145k2020_20220617_06 = UnfilteredUserInterestedIn_Model20m145k2020_Default.copy( maxTweetCandidateAge = 48.hours, ) // Experimental SANN config private val UnfilteredUserInterestedIn_Model20m145k2020_20220801 = UnfilteredUserInterestedIn_Model20m145k2020_20220617_06.copy( candidateEmbeddingType = EmbeddingType.VideoPlayBack50LogFavBasedTweet, ) // SANN-1 config private val UnfilteredUserInterestedIn_Model20m145k2020_20220810 = UnfilteredUserInterestedIn_Model20m145k2020_Default.copy( maxNumResults = 100, candidateEmbeddingType = EmbeddingType.LogFavBasedAdsTweet, maxTweetCandidateAge = 175200.hours, maxTopTweetsPerCluster = 1600 ) // SANN-2 config private val UnfilteredUserInterestedIn_Model20m145k2020_20220818 = UnfilteredUserInterestedIn_Model20m145k2020_Default.copy( maxNumResults = 100, candidateEmbeddingType = EmbeddingType.LogFavClickBasedAdsTweet, maxTweetCandidateAge = 175200.hours, maxTopTweetsPerCluster = 1600 ) // SANN-3 config private val UnfilteredUserInterestedIn_Model20m145k2020_20220819 = UnfilteredUserInterestedIn_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.PushOpenLogFavBasedTweet, ) // SANN-5 config private val UnfilteredUserInterestedIn_Model20m145k2020_20221221 = UnfilteredUserInterestedIn_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.LogFavBasedRealTimeTweet, maxTweetCandidateAge = 1.hours ) // SANN-4 config private val UnfilteredUserInterestedIn_Model20m145k2020_20221220 = UnfilteredUserInterestedIn_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.LogFavBasedEvergreenTweet, maxTweetCandidateAge = 48.hours ) private val LogFavBasedUserInterestedInFromAPE_Model20m145k2020_Default = DefaultConfig.copy() // Chunnan's exp on maxTweetCandidateAgeDays 2 private val LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20220617_06 = LogFavBasedUserInterestedInFromAPE_Model20m145k2020_Default.copy( maxTweetCandidateAge = 48.hours, ) // Experimental SANN config private val LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20220801 = LogFavBasedUserInterestedInFromAPE_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.VideoPlayBack50LogFavBasedTweet, ) // SANN-1 config private val LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20220810 = LogFavBasedUserInterestedInFromAPE_Model20m145k2020_Default.copy( maxNumResults = 100, candidateEmbeddingType = EmbeddingType.LogFavBasedAdsTweet, maxTweetCandidateAge = 175200.hours, maxTopTweetsPerCluster = 1600 ) // SANN-2 config private val LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20220818 = LogFavBasedUserInterestedInFromAPE_Model20m145k2020_Default.copy( maxNumResults = 100, candidateEmbeddingType = EmbeddingType.LogFavClickBasedAdsTweet, maxTweetCandidateAge = 175200.hours, maxTopTweetsPerCluster = 1600 ) // SANN-3 config private val LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20220819 = LogFavBasedUserInterestedInFromAPE_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.PushOpenLogFavBasedTweet, ) // SANN-5 config private val LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20221221 = LogFavBasedUserInterestedInFromAPE_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.LogFavBasedRealTimeTweet, maxTweetCandidateAge = 1.hours ) // SANN-4 config private val LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20221220 = LogFavBasedUserInterestedInFromAPE_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.LogFavBasedEvergreenTweet, maxTweetCandidateAge = 48.hours ) private val LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_Default = DefaultConfig.copy() // Chunnan's exp on maxTweetCandidateAgeDays 2 private val LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20220617_06 = LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_Default.copy( maxTweetCandidateAge = 48.hours, ) // Experimental SANN config private val LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20220801 = LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.VideoPlayBack50LogFavBasedTweet, ) // SANN-1 config private val LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20220810 = LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_Default.copy( maxNumResults = 100, candidateEmbeddingType = EmbeddingType.LogFavBasedAdsTweet, maxTweetCandidateAge = 175200.hours, maxTopTweetsPerCluster = 1600 ) // SANN-2 config private val LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20220818 = LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_Default.copy( maxNumResults = 100, candidateEmbeddingType = EmbeddingType.LogFavClickBasedAdsTweet, maxTweetCandidateAge = 175200.hours, maxTopTweetsPerCluster = 1600 ) // SANN-3 config private val LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20220819 = LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.PushOpenLogFavBasedTweet, ) // SANN-5 config private val LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20221221 = LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.LogFavBasedRealTimeTweet, maxTweetCandidateAge = 1.hours ) // SANN-4 config private val LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20221220 = LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.LogFavBasedEvergreenTweet, maxTweetCandidateAge = 48.hours ) private val UserNextInterestedIn_Model20m145k2020_Default = DefaultConfig.copy() // Chunnan's exp on maxTweetCandidateAgeDays 2 private val UserNextInterestedIn_Model20m145k2020_20220617_06 = UserNextInterestedIn_Model20m145k2020_Default.copy( maxTweetCandidateAge = 48.hours, ) // Experimental SANN config private val UserNextInterestedIn_Model20m145k2020_20220801 = UserNextInterestedIn_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.VideoPlayBack50LogFavBasedTweet, ) // SANN-1 config private val UserNextInterestedIn_Model20m145k2020_20220810 = UserNextInterestedIn_Model20m145k2020_Default.copy( maxNumResults = 100, candidateEmbeddingType = EmbeddingType.LogFavBasedAdsTweet, maxTweetCandidateAge = 175200.hours, maxTopTweetsPerCluster = 1600 ) // SANN-2 config private val UserNextInterestedIn_Model20m145k2020_20220818 = UserNextInterestedIn_Model20m145k2020_Default.copy( maxNumResults = 100, candidateEmbeddingType = EmbeddingType.LogFavClickBasedAdsTweet, maxTweetCandidateAge = 175200.hours, maxTopTweetsPerCluster = 1600 ) // SANN-3 config private val UserNextInterestedIn_Model20m145k2020_20220819 = UserNextInterestedIn_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.PushOpenLogFavBasedTweet, ) // SANN-5 config private val UserNextInterestedIn_Model20m145k2020_20221221 = UserNextInterestedIn_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.LogFavBasedRealTimeTweet, maxTweetCandidateAge = 1.hours ) // SANN-4 config private val UserNextInterestedIn_Model20m145k2020_20221220 = UserNextInterestedIn_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.LogFavBasedEvergreenTweet, maxTweetCandidateAge = 48.hours ) // Vincent's experiment on using FollowBasedProducer as query embedding type for UserFollow private val FollowBasedProducer_Model20m145k2020_Default = FavBasedProducer_Model20m145k2020_Default.copy() // Experimental SANN config private val FollowBasedProducer_Model20m145k2020_20220801 = FavBasedProducer_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.VideoPlayBack50LogFavBasedTweet, ) // SANN-1 config private val FollowBasedProducer_Model20m145k2020_20220810 = FavBasedProducer_Model20m145k2020_Default.copy( maxNumResults = 100, candidateEmbeddingType = EmbeddingType.LogFavBasedAdsTweet, maxTweetCandidateAge = 175200.hours, maxTopTweetsPerCluster = 1600 ) // SANN-2 config private val FollowBasedProducer_Model20m145k2020_20220818 = FavBasedProducer_Model20m145k2020_Default.copy( maxNumResults = 100, candidateEmbeddingType = EmbeddingType.LogFavClickBasedAdsTweet, maxTweetCandidateAge = 175200.hours, maxTopTweetsPerCluster = 1600 ) // SANN-3 config private val FollowBasedProducer_Model20m145k2020_20220819 = FavBasedProducer_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.PushOpenLogFavBasedTweet, ) // SANN-5 config private val FollowBasedProducer_Model20m145k2020_20221221 = FavBasedProducer_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.LogFavBasedRealTimeTweet, maxTweetCandidateAge = 1.hours ) // SANN-4 config private val FollowBasedProducer_Model20m145k2020_20221220 = FavBasedProducer_Model20m145k2020_Default.copy( candidateEmbeddingType = EmbeddingType.LogFavBasedEvergreenTweet, maxTweetCandidateAge = 48.hours ) val DefaultConfigMappings: Map[String, SimClustersANNConfig] = Map( "FavBasedProducer_Model20m145k2020_Default" -> FavBasedProducer_Model20m145k2020_Default, "FavBasedProducer_Model20m145k2020_20220617_06" -> FavBasedProducer_Model20m145k2020_20220617_06, "FavBasedProducer_Model20m145k2020_20220801" -> FavBasedProducer_Model20m145k2020_20220801, "FavBasedProducer_Model20m145k2020_20220810" -> FavBasedProducer_Model20m145k2020_20220810, "FavBasedProducer_Model20m145k2020_20220818" -> FavBasedProducer_Model20m145k2020_20220818, "FavBasedProducer_Model20m145k2020_20220819" -> FavBasedProducer_Model20m145k2020_20220819, "FavBasedProducer_Model20m145k2020_20221221" -> FavBasedProducer_Model20m145k2020_20221221, "FavBasedProducer_Model20m145k2020_20221220" -> FavBasedProducer_Model20m145k2020_20221220, "FollowBasedProducer_Model20m145k2020_Default" -> FollowBasedProducer_Model20m145k2020_Default, "FollowBasedProducer_Model20m145k2020_20220801" -> FollowBasedProducer_Model20m145k2020_20220801, "FollowBasedProducer_Model20m145k2020_20220810" -> FollowBasedProducer_Model20m145k2020_20220810, "FollowBasedProducer_Model20m145k2020_20220818" -> FollowBasedProducer_Model20m145k2020_20220818, "FollowBasedProducer_Model20m145k2020_20220819" -> FollowBasedProducer_Model20m145k2020_20220819, "FollowBasedProducer_Model20m145k2020_20221221" -> FollowBasedProducer_Model20m145k2020_20221221, "FollowBasedProducer_Model20m145k2020_20221220" -> FollowBasedProducer_Model20m145k2020_20221220, "LogFavLongestL2EmbeddingTweet_Model20m145k2020_Default" -> LogFavLongestL2EmbeddingTweet_Model20m145k2020_Default, "LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220617_06" -> LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220617_06, "LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220801" -> LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220801, "LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220810" -> LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220810, "LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220818" -> LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220818, "LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220819" -> LogFavLongestL2EmbeddingTweet_Model20m145k2020_20220819, "LogFavLongestL2EmbeddingTweet_Model20m145k2020_20221221" -> LogFavLongestL2EmbeddingTweet_Model20m145k2020_20221221, "LogFavLongestL2EmbeddingTweet_Model20m145k2020_20221220" -> LogFavLongestL2EmbeddingTweet_Model20m145k2020_20221220, "UnfilteredUserInterestedIn_Model20m145k2020_Default" -> UnfilteredUserInterestedIn_Model20m145k2020_Default, "UnfilteredUserInterestedIn_Model20m145k2020_20220617_06" -> UnfilteredUserInterestedIn_Model20m145k2020_20220617_06, "UnfilteredUserInterestedIn_Model20m145k2020_20220801" -> UnfilteredUserInterestedIn_Model20m145k2020_20220801, "UnfilteredUserInterestedIn_Model20m145k2020_20220810" -> UnfilteredUserInterestedIn_Model20m145k2020_20220810, "UnfilteredUserInterestedIn_Model20m145k2020_20220818" -> UnfilteredUserInterestedIn_Model20m145k2020_20220818, "UnfilteredUserInterestedIn_Model20m145k2020_20220819" -> UnfilteredUserInterestedIn_Model20m145k2020_20220819, "UnfilteredUserInterestedIn_Model20m145k2020_20221221" -> UnfilteredUserInterestedIn_Model20m145k2020_20221221, "UnfilteredUserInterestedIn_Model20m145k2020_20221220" -> UnfilteredUserInterestedIn_Model20m145k2020_20221220, "LogFavBasedUserInterestedInFromAPE_Model20m145k2020_Default" -> LogFavBasedUserInterestedInFromAPE_Model20m145k2020_Default, "LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20220617_06" -> LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20220617_06, "LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20220801" -> LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20220801, "LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20220810" -> LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20220810, "LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20220818" -> LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20220818, "LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20220819" -> LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20220819, "LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20221221" -> LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20221221, "LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20221220" -> LogFavBasedUserInterestedInFromAPE_Model20m145k2020_20221220, "LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_Default" -> LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_Default, "LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20220617_06" -> LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20220617_06, "LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20220801" -> LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20220801, "LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20220810" -> LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20220810, "LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20220818" -> LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20220818, "LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20220819" -> LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20220819, "LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20221221" -> LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20221221, "LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20221220" -> LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020_20221220, "UserNextInterestedIn_Model20m145k2020_Default" -> UserNextInterestedIn_Model20m145k2020_Default, "UserNextInterestedIn_Model20m145k2020_20220617_06" -> UserNextInterestedIn_Model20m145k2020_20220617_06, "UserNextInterestedIn_Model20m145k2020_20220801" -> UserNextInterestedIn_Model20m145k2020_20220801, "UserNextInterestedIn_Model20m145k2020_20220810" -> UserNextInterestedIn_Model20m145k2020_20220810, "UserNextInterestedIn_Model20m145k2020_20220818" -> UserNextInterestedIn_Model20m145k2020_20220818, "UserNextInterestedIn_Model20m145k2020_20220819" -> UserNextInterestedIn_Model20m145k2020_20220819, "UserNextInterestedIn_Model20m145k2020_20221221" -> UserNextInterestedIn_Model20m145k2020_20221221, "UserNextInterestedIn_Model20m145k2020_20221220" -> UserNextInterestedIn_Model20m145k2020_20221220, ) def getConfig( embeddingType: String, modelVersion: String, id: String ): SimClustersANNConfig = { val configName = embeddingType + "_" + modelVersion + "_" + id DefaultConfigMappings.get(configName) match { case Some(config) => config case None => throw InvalidSANNConfigException(s"Incorrect config id passed in for SANN $configName") } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/config/TimeoutConfig.scala ================================================ package com.twitter.cr_mixer.config import com.twitter.util.Duration case class TimeoutConfig( /* Default timeouts for candidate generator */ serviceTimeout: Duration, signalFetchTimeout: Duration, similarityEngineTimeout: Duration, annServiceClientTimeout: Duration, /* For Uteg Candidate Generator */ utegSimilarityEngineTimeout: Duration, /* For User State Store */ userStateUnderlyingStoreTimeout: Duration, userStateStoreTimeout: Duration, /* For FRS based tweets */ // Timeout passed to EarlyBird server earlybirdServerTimeout: Duration, // Timeout set on CrMixer side earlybirdSimilarityEngineTimeout: Duration, frsBasedTweetEndpointTimeout: Duration, topicTweetEndpointTimeout: Duration, // Timeout Settings for Navi gRPC Client naviRequestTimeout: Duration) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/controller/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/storehaus:core", "3rdparty/src/jvm/com/twitter/storehaus:core", "content-recommender/thrift/src/main/thrift:content-recommender-common-scala", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/debug", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/featureswitch", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util", "cr-mixer/thrift/src/main/thrift:thrift-scala", "finagle/finagle-base-http/src/main", "finagle/finagle-core/src/main", "finagle/finagle-http/src/main/scala", "finatra/http-server/src/main/scala/com/twitter/finatra/http:controller", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift:controller", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/base", "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", "simclusters-ann/thrift/src/main/thrift:thrift-scala", "src/scala/com/twitter/simclusters_v2/common", "src/thrift/com/twitter/ads/schema:common-scala", "src/thrift/com/twitter/context:twitter-context-scala", "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", "src/thrift/com/twitter/frigate/data_pipeline/scalding:blue_verified_annotations-scala", "src/thrift/com/twitter/onboarding/relevance/coldstart_lookalike:coldstartlookalike-thrift-scala", "src/thrift/com/twitter/recos:recos-common-scala", "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", "src/thrift/com/twitter/timelines/render:thrift-scala", "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", "stringcenter/client", "timelines/src/main/scala/com/twitter/timelines/tracing/lensview", "timelines/src/main/scala/com/twitter/timelines/tracing/lensview/funnelseries", "twitter-context/src/main/scala", "user-signal-service/thrift/src/main/thrift:thrift-scala", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/controller/CrMixerThriftController.scala ================================================ package com.twitter.cr_mixer.controller import com.twitter.core_workflows.user_model.thriftscala.UserState import com.twitter.cr_mixer.candidate_generation.AdsCandidateGenerator import com.twitter.cr_mixer.candidate_generation.CrCandidateGenerator import com.twitter.cr_mixer.candidate_generation.FrsTweetCandidateGenerator import com.twitter.cr_mixer.candidate_generation.RelatedTweetCandidateGenerator import com.twitter.cr_mixer.candidate_generation.RelatedVideoTweetCandidateGenerator import com.twitter.cr_mixer.candidate_generation.TopicTweetCandidateGenerator import com.twitter.cr_mixer.candidate_generation.UtegTweetCandidateGenerator import com.twitter.cr_mixer.featureswitch.ParamsBuilder import com.twitter.cr_mixer.logging.CrMixerScribeLogger import com.twitter.cr_mixer.logging.RelatedTweetScribeLogger import com.twitter.cr_mixer.logging.AdsRecommendationsScribeLogger import com.twitter.cr_mixer.logging.RelatedTweetScribeMetadata import com.twitter.cr_mixer.logging.ScribeMetadata import com.twitter.cr_mixer.logging.UtegTweetScribeLogger import com.twitter.cr_mixer.model.AdsCandidateGeneratorQuery import com.twitter.cr_mixer.model.CrCandidateGeneratorQuery import com.twitter.cr_mixer.model.FrsTweetCandidateGeneratorQuery import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.model.RankedAdsCandidate import com.twitter.cr_mixer.model.RankedCandidate import com.twitter.cr_mixer.model.RelatedTweetCandidateGeneratorQuery import com.twitter.cr_mixer.model.RelatedVideoTweetCandidateGeneratorQuery import com.twitter.cr_mixer.model.TopicTweetCandidateGeneratorQuery import com.twitter.cr_mixer.model.TweetWithScoreAndSocialProof import com.twitter.cr_mixer.model.UtegTweetCandidateGeneratorQuery import com.twitter.cr_mixer.param.AdsParams import com.twitter.cr_mixer.param.FrsParams.FrsBasedCandidateGenerationMaxCandidatesNumParam import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.param.RelatedTweetGlobalParams import com.twitter.cr_mixer.param.RelatedVideoTweetGlobalParams import com.twitter.cr_mixer.param.TopicTweetParams import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.param.decider.EndpointLoadShedder import com.twitter.cr_mixer.thriftscala.AdTweetRecommendation import com.twitter.cr_mixer.thriftscala.AdsRequest import com.twitter.cr_mixer.thriftscala.AdsResponse import com.twitter.cr_mixer.thriftscala.CrMixerTweetRequest import com.twitter.cr_mixer.thriftscala.CrMixerTweetResponse import com.twitter.cr_mixer.thriftscala.FrsTweetRequest import com.twitter.cr_mixer.thriftscala.FrsTweetResponse import com.twitter.cr_mixer.thriftscala.RelatedTweet import com.twitter.cr_mixer.thriftscala.RelatedTweetRequest import com.twitter.cr_mixer.thriftscala.RelatedTweetResponse import com.twitter.cr_mixer.thriftscala.RelatedVideoTweet import com.twitter.cr_mixer.thriftscala.RelatedVideoTweetRequest import com.twitter.cr_mixer.thriftscala.RelatedVideoTweetResponse import com.twitter.cr_mixer.thriftscala.TopicTweet import com.twitter.cr_mixer.thriftscala.TopicTweetRequest import com.twitter.cr_mixer.thriftscala.TopicTweetResponse import com.twitter.cr_mixer.thriftscala.TweetRecommendation import com.twitter.cr_mixer.thriftscala.UtegTweet import com.twitter.cr_mixer.thriftscala.UtegTweetRequest import com.twitter.cr_mixer.thriftscala.UtegTweetResponse import com.twitter.cr_mixer.util.MetricTagUtil import com.twitter.cr_mixer.util.SignalTimestampStatsUtil import com.twitter.cr_mixer.{thriftscala => t} import com.twitter.finagle.stats.StatsReceiver import com.twitter.finatra.thrift.Controller import com.twitter.hermit.store.common.ReadableWritableStore import com.twitter.simclusters_v2.common.UserId import com.twitter.simclusters_v2.thriftscala.TopicId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog} import com.twitter.timelines.tracing.lensview.funnelseries.TweetScoreFunnelSeries import com.twitter.util.Future import com.twitter.util.Time import java.util.UUID import javax.inject.Inject import org.apache.commons.lang.exception.ExceptionUtils class CrMixerThriftController @Inject() ( crCandidateGenerator: CrCandidateGenerator, relatedTweetCandidateGenerator: RelatedTweetCandidateGenerator, relatedVideoTweetCandidateGenerator: RelatedVideoTweetCandidateGenerator, utegTweetCandidateGenerator: UtegTweetCandidateGenerator, frsTweetCandidateGenerator: FrsTweetCandidateGenerator, topicTweetCandidateGenerator: TopicTweetCandidateGenerator, crMixerScribeLogger: CrMixerScribeLogger, relatedTweetScribeLogger: RelatedTweetScribeLogger, utegTweetScribeLogger: UtegTweetScribeLogger, adsRecommendationsScribeLogger: AdsRecommendationsScribeLogger, adsCandidateGenerator: AdsCandidateGenerator, decider: CrMixerDecider, paramsBuilder: ParamsBuilder, endpointLoadShedder: EndpointLoadShedder, signalTimestampStatsUtil: SignalTimestampStatsUtil, tweetRecommendationResultsStore: ReadableWritableStore[UserId, CrMixerTweetResponse], userStateStore: ReadableStore[UserId, UserState], statsReceiver: StatsReceiver) extends Controller(t.CrMixer) { lazy private val tweetScoreFunnelSeries = new TweetScoreFunnelSeries(statsReceiver) private def logErrMessage(endpoint: String, e: Throwable): Unit = { val msg = Seq( s"Failed endpoint $endpoint: ${e.getLocalizedMessage}", ExceptionUtils.getStackTrace(e) ).mkString("\n") /** * * We chose logger.info() here to print message instead of logger.error since that * logger.error sometimes suppresses detailed stacktrace. */ logger.info(msg) } private def generateRequestUUID(): Long = { /** * * We generate unique UUID via bitwise operations. See the below link for more: * https://stackoverflow.com/questions/15184820/how-to-generate-unique-positive-long-using-uuid */ UUID.randomUUID().getMostSignificantBits & Long.MaxValue } handle(t.CrMixer.GetTweetRecommendations) { args: t.CrMixer.GetTweetRecommendations.Args => val endpointName = "getTweetRecommendations" val requestUUID = generateRequestUUID() val startTime = Time.now.inMilliseconds val userId = args.request.clientContext.userId.getOrElse( throw new IllegalArgumentException("userId must be present in the Thrift clientContext") ) val queryFut = buildCrCandidateGeneratorQuery(args.request, requestUUID, userId) queryFut.flatMap { query => val scribeMetadata = ScribeMetadata.from(query) endpointLoadShedder(endpointName, query.product.originalName) { val response = crCandidateGenerator.get(query) val blueVerifiedScribedResponse = response.flatMap { rankedCandidates => val hasBlueVerifiedCandidate = rankedCandidates.exists { tweet => tweet.tweetInfo.hasBlueVerifiedAnnotation.contains(true) } if (hasBlueVerifiedCandidate) { crMixerScribeLogger.scribeGetTweetRecommendationsForBlueVerified( scribeMetadata, response) } else { response } } val thriftResponse = blueVerifiedScribedResponse.map { candidates => if (query.product == t.Product.Home) { scribeTweetScoreFunnelSeries(candidates) } buildThriftResponse(candidates) } cacheTweetRecommendationResults(args.request, thriftResponse) crMixerScribeLogger.scribeGetTweetRecommendations( args.request, startTime, scribeMetadata, thriftResponse) }.rescue { case EndpointLoadShedder.LoadSheddingException => Future(CrMixerTweetResponse(Seq.empty)) case e => logErrMessage(endpointName, e) Future(CrMixerTweetResponse(Seq.empty)) } } } /** * * GetRelatedTweetsForQueryTweet and GetRelatedTweetsForQueryAuthor are essentially * doing very similar things, except that one passes in TweetId which calls TweetBased engine, * and the other passes in AuthorId which calls ProducerBased engine. */ handle(t.CrMixer.GetRelatedTweetsForQueryTweet) { args: t.CrMixer.GetRelatedTweetsForQueryTweet.Args => val endpointName = "getRelatedTweetsForQueryTweet" getRelatedTweets(endpointName, args.request) } handle(t.CrMixer.GetRelatedVideoTweetsForQueryTweet) { args: t.CrMixer.GetRelatedVideoTweetsForQueryTweet.Args => val endpointName = "getRelatedVideoTweetsForQueryVideoTweet" getRelatedVideoTweets(endpointName, args.request) } handle(t.CrMixer.GetRelatedTweetsForQueryAuthor) { args: t.CrMixer.GetRelatedTweetsForQueryAuthor.Args => val endpointName = "getRelatedTweetsForQueryAuthor" getRelatedTweets(endpointName, args.request) } private def getRelatedTweets( endpointName: String, request: RelatedTweetRequest ): Future[RelatedTweetResponse] = { val requestUUID = generateRequestUUID() val startTime = Time.now.inMilliseconds val queryFut = buildRelatedTweetQuery(request, requestUUID) queryFut.flatMap { query => val relatedTweetScribeMetadata = RelatedTweetScribeMetadata.from(query) endpointLoadShedder(endpointName, query.product.originalName) { relatedTweetScribeLogger.scribeGetRelatedTweets( request, startTime, relatedTweetScribeMetadata, relatedTweetCandidateGenerator .get(query) .map(buildRelatedTweetResponse)) }.rescue { case EndpointLoadShedder.LoadSheddingException => Future(RelatedTweetResponse(Seq.empty)) case e => logErrMessage(endpointName, e) Future(RelatedTweetResponse(Seq.empty)) } } } private def getRelatedVideoTweets( endpointName: String, request: RelatedVideoTweetRequest ): Future[RelatedVideoTweetResponse] = { val requestUUID = generateRequestUUID() val queryFut = buildRelatedVideoTweetQuery(request, requestUUID) queryFut.flatMap { query => endpointLoadShedder(endpointName, query.product.originalName) { relatedVideoTweetCandidateGenerator.get(query).map { initialCandidateSeq => buildRelatedVideoTweetResponse(initialCandidateSeq) } }.rescue { case EndpointLoadShedder.LoadSheddingException => Future(RelatedVideoTweetResponse(Seq.empty)) case e => logErrMessage(endpointName, e) Future(RelatedVideoTweetResponse(Seq.empty)) } } } handle(t.CrMixer.GetFrsBasedTweetRecommendations) { args: t.CrMixer.GetFrsBasedTweetRecommendations.Args => val endpointName = "getFrsBasedTweetRecommendations" val requestUUID = generateRequestUUID() val queryFut = buildFrsBasedTweetQuery(args.request, requestUUID) queryFut.flatMap { query => endpointLoadShedder(endpointName, query.product.originalName) { frsTweetCandidateGenerator.get(query).map(FrsTweetResponse(_)) }.rescue { case e => logErrMessage(endpointName, e) Future(FrsTweetResponse(Seq.empty)) } } } handle(t.CrMixer.GetTopicTweetRecommendations) { args: t.CrMixer.GetTopicTweetRecommendations.Args => val endpointName = "getTopicTweetRecommendations" val requestUUID = generateRequestUUID() val query = buildTopicTweetQuery(args.request, requestUUID) endpointLoadShedder(endpointName, query.product.originalName) { topicTweetCandidateGenerator.get(query).map(TopicTweetResponse(_)) }.rescue { case e => logErrMessage(endpointName, e) Future(TopicTweetResponse(Map.empty[Long, Seq[TopicTweet]])) } } handle(t.CrMixer.GetUtegTweetRecommendations) { args: t.CrMixer.GetUtegTweetRecommendations.Args => val endpointName = "getUtegTweetRecommendations" val requestUUID = generateRequestUUID() val startTime = Time.now.inMilliseconds val queryFut = buildUtegTweetQuery(args.request, requestUUID) queryFut .flatMap { query => val scribeMetadata = ScribeMetadata.from(query) endpointLoadShedder(endpointName, query.product.originalName) { utegTweetScribeLogger.scribeGetUtegTweetRecommendations( args.request, startTime, scribeMetadata, utegTweetCandidateGenerator .get(query) .map(buildUtegTweetResponse) ) }.rescue { case e => logErrMessage(endpointName, e) Future(UtegTweetResponse(Seq.empty)) } } } handle(t.CrMixer.GetAdsRecommendations) { args: t.CrMixer.GetAdsRecommendations.Args => val endpointName = "getAdsRecommendations" val queryFut = buildAdsCandidateGeneratorQuery(args.request) val startTime = Time.now.inMilliseconds queryFut.flatMap { query => { val scribeMetadata = ScribeMetadata.from(query) val response = adsCandidateGenerator .get(query).map { candidates => buildAdsResponse(candidates) } adsRecommendationsScribeLogger.scribeGetAdsRecommendations( args.request, startTime, scribeMetadata, response, query.params(AdsParams.EnableScribe) ) }.rescue { case e => logErrMessage(endpointName, e) Future(AdsResponse(Seq.empty)) } } } private def buildCrCandidateGeneratorQuery( thriftRequest: CrMixerTweetRequest, requestUUID: Long, userId: Long ): Future[CrCandidateGeneratorQuery] = { val product = thriftRequest.product val productContext = thriftRequest.productContext val scopedStats = statsReceiver .scope(product.toString).scope("CrMixerTweetRequest") userStateStore .get(userId).map { userStateOpt => val userState = userStateOpt .getOrElse(UserState.EnumUnknownUserState(100)) scopedStats.scope("UserState").counter(userState.toString).incr() val params = paramsBuilder.buildFromClientContext( thriftRequest.clientContext, thriftRequest.product, userState ) // Specify product-specific behavior mapping here val maxNumResults = (product, productContext) match { case (t.Product.Home, Some(t.ProductContext.HomeContext(homeContext))) => homeContext.maxResults.getOrElse(9999) case (t.Product.Notifications, Some(t.ProductContext.NotificationsContext(cxt))) => params(GlobalParams.MaxCandidatesPerRequestParam) case (t.Product.Email, None) => params(GlobalParams.MaxCandidatesPerRequestParam) case (t.Product.ImmersiveMediaViewer, None) => params(GlobalParams.MaxCandidatesPerRequestParam) case (t.Product.VideoCarousel, None) => params(GlobalParams.MaxCandidatesPerRequestParam) case _ => throw new IllegalArgumentException( s"Product ${product} and ProductContext ${productContext} are not allowed in CrMixer" ) } CrCandidateGeneratorQuery( userId = userId, product = product, userState = userState, maxNumResults = maxNumResults, impressedTweetList = thriftRequest.excludedTweetIds.getOrElse(Nil).toSet, params = params, requestUUID = requestUUID, languageCode = thriftRequest.clientContext.languageCode ) } } private def buildRelatedTweetQuery( thriftRequest: RelatedTweetRequest, requestUUID: Long ): Future[RelatedTweetCandidateGeneratorQuery] = { val product = thriftRequest.product val scopedStats = statsReceiver .scope(product.toString).scope("RelatedTweetRequest") val userStateFut: Future[UserState] = (thriftRequest.clientContext.userId match { case Some(userId) => userStateStore.get(userId) case None => Future.value(Some(UserState.EnumUnknownUserState(100))) }).map(_.getOrElse(UserState.EnumUnknownUserState(100))) userStateFut.map { userState => scopedStats.scope("UserState").counter(userState.toString).incr() val params = paramsBuilder.buildFromClientContext( thriftRequest.clientContext, thriftRequest.product, userState) // Specify product-specific behavior mapping here // Currently, Home takes 10, and RUX takes 100 val maxNumResults = params(RelatedTweetGlobalParams.MaxCandidatesPerRequestParam) RelatedTweetCandidateGeneratorQuery( internalId = thriftRequest.internalId, clientContext = thriftRequest.clientContext, product = product, maxNumResults = maxNumResults, impressedTweetList = thriftRequest.excludedTweetIds.getOrElse(Nil).toSet, params = params, requestUUID = requestUUID ) } } private def buildAdsCandidateGeneratorQuery( thriftRequest: AdsRequest ): Future[AdsCandidateGeneratorQuery] = { val userId = thriftRequest.clientContext.userId.getOrElse( throw new IllegalArgumentException("userId must be present in the Thrift clientContext") ) val product = thriftRequest.product val requestUUID = generateRequestUUID() userStateStore .get(userId).map { userStateOpt => val userState = userStateOpt .getOrElse(UserState.EnumUnknownUserState(100)) val params = paramsBuilder.buildFromClientContext( thriftRequest.clientContext, thriftRequest.product, userState) val maxNumResults = params(AdsParams.AdsCandidateGenerationMaxCandidatesNumParam) AdsCandidateGeneratorQuery( userId = userId, product = product, userState = userState, params = params, maxNumResults = maxNumResults, requestUUID = requestUUID ) } } private def buildRelatedVideoTweetQuery( thriftRequest: RelatedVideoTweetRequest, requestUUID: Long ): Future[RelatedVideoTweetCandidateGeneratorQuery] = { val product = thriftRequest.product val scopedStats = statsReceiver .scope(product.toString).scope("RelatedVideoTweetRequest") val userStateFut: Future[UserState] = (thriftRequest.clientContext.userId match { case Some(userId) => userStateStore.get(userId) case None => Future.value(Some(UserState.EnumUnknownUserState(100))) }).map(_.getOrElse(UserState.EnumUnknownUserState(100))) userStateFut.map { userState => scopedStats.scope("UserState").counter(userState.toString).incr() val params = paramsBuilder.buildFromClientContext( thriftRequest.clientContext, thriftRequest.product, userState) val maxNumResults = params(RelatedVideoTweetGlobalParams.MaxCandidatesPerRequestParam) RelatedVideoTweetCandidateGeneratorQuery( internalId = thriftRequest.internalId, clientContext = thriftRequest.clientContext, product = product, maxNumResults = maxNumResults, impressedTweetList = thriftRequest.excludedTweetIds.getOrElse(Nil).toSet, params = params, requestUUID = requestUUID ) } } private def buildUtegTweetQuery( thriftRequest: UtegTweetRequest, requestUUID: Long ): Future[UtegTweetCandidateGeneratorQuery] = { val userId = thriftRequest.clientContext.userId.getOrElse( throw new IllegalArgumentException("userId must be present in the Thrift clientContext") ) val product = thriftRequest.product val productContext = thriftRequest.productContext val scopedStats = statsReceiver .scope(product.toString).scope("UtegTweetRequest") userStateStore .get(userId).map { userStateOpt => val userState = userStateOpt .getOrElse(UserState.EnumUnknownUserState(100)) scopedStats.scope("UserState").counter(userState.toString).incr() val params = paramsBuilder.buildFromClientContext( thriftRequest.clientContext, thriftRequest.product, userState ) // Specify product-specific behavior mapping here val maxNumResults = (product, productContext) match { case (t.Product.Home, Some(t.ProductContext.HomeContext(homeContext))) => homeContext.maxResults.getOrElse(9999) case _ => throw new IllegalArgumentException( s"Product ${product} and ProductContext ${productContext} are not allowed in CrMixer" ) } UtegTweetCandidateGeneratorQuery( userId = userId, product = product, userState = userState, maxNumResults = maxNumResults, impressedTweetList = thriftRequest.excludedTweetIds.getOrElse(Nil).toSet, params = params, requestUUID = requestUUID ) } } private def buildTopicTweetQuery( thriftRequest: TopicTweetRequest, requestUUID: Long ): TopicTweetCandidateGeneratorQuery = { val userId = thriftRequest.clientContext.userId.getOrElse( throw new IllegalArgumentException( "userId must be present in the TopicTweetRequest clientContext") ) val product = thriftRequest.product val productContext = thriftRequest.productContext // Specify product-specific behavior mapping here val isVideoOnly = (product, productContext) match { case (t.Product.ExploreTopics, Some(t.ProductContext.ExploreContext(context))) => context.isVideoOnly case (t.Product.TopicLandingPage, None) => false case (t.Product.HomeTopicsBackfill, None) => false case (t.Product.TopicTweetsStrato, None) => false case _ => throw new IllegalArgumentException( s"Product ${product} and ProductContext ${productContext} are not allowed in CrMixer" ) } statsReceiver.scope(product.toString).counter(TopicTweetRequest.toString).incr() val params = paramsBuilder.buildFromClientContext( thriftRequest.clientContext, product, UserState.EnumUnknownUserState(100) ) val topicIds = thriftRequest.topicIds.map { topicId => TopicId( entityId = topicId, language = thriftRequest.clientContext.languageCode, country = None ) }.toSet TopicTweetCandidateGeneratorQuery( userId = userId, topicIds = topicIds, product = product, maxNumResults = params(TopicTweetParams.MaxTopicTweetCandidatesParam), impressedTweetList = thriftRequest.excludedTweetIds.getOrElse(Nil).toSet, params = params, requestUUID = requestUUID, isVideoOnly = isVideoOnly ) } private def buildFrsBasedTweetQuery( thriftRequest: FrsTweetRequest, requestUUID: Long ): Future[FrsTweetCandidateGeneratorQuery] = { val userId = thriftRequest.clientContext.userId.getOrElse( throw new IllegalArgumentException( "userId must be present in the FrsTweetRequest clientContext") ) val product = thriftRequest.product val productContext = thriftRequest.productContext val scopedStats = statsReceiver .scope(product.toString).scope("FrsTweetRequest") userStateStore .get(userId).map { userStateOpt => val userState = userStateOpt .getOrElse(UserState.EnumUnknownUserState(100)) scopedStats.scope("UserState").counter(userState.toString).incr() val params = paramsBuilder.buildFromClientContext( thriftRequest.clientContext, thriftRequest.product, userState ) val maxNumResults = (product, productContext) match { case (t.Product.Home, Some(t.ProductContext.HomeContext(homeContext))) => homeContext.maxResults.getOrElse( params(FrsBasedCandidateGenerationMaxCandidatesNumParam)) case _ => params(FrsBasedCandidateGenerationMaxCandidatesNumParam) } FrsTweetCandidateGeneratorQuery( userId = userId, product = product, maxNumResults = maxNumResults, impressedTweetList = thriftRequest.excludedTweetIds.getOrElse(Nil).toSet, impressedUserList = thriftRequest.excludedUserIds.getOrElse(Nil).toSet, params = params, languageCodeOpt = thriftRequest.clientContext.languageCode, countryCodeOpt = thriftRequest.clientContext.countryCode, requestUUID = requestUUID ) } } private def buildThriftResponse( candidates: Seq[RankedCandidate] ): CrMixerTweetResponse = { val tweets = candidates.map { candidate => TweetRecommendation( tweetId = candidate.tweetId, score = candidate.predictionScore, metricTags = Some(MetricTagUtil.buildMetricTags(candidate)), latestSourceSignalTimestampInMillis = SignalTimestampStatsUtil.buildLatestSourceSignalTimestamp(candidate) ) } signalTimestampStatsUtil.statsSignalTimestamp(tweets) CrMixerTweetResponse(tweets) } private def scribeTweetScoreFunnelSeries( candidates: Seq[RankedCandidate] ): Seq[RankedCandidate] = { // 202210210901 is a random number for code search of Lensview tweetScoreFunnelSeries.startNewSpan( name = "GetTweetRecommendationsTopLevelTweetSimilarityEngineType", codePtr = 202210210901L) { ( candidates, candidates.map { candidate => thriftlog.TweetDimensionMeasure( dimension = Some( thriftlog .RequestTweetDimension( candidate.tweetId, candidate.reasonChosen.similarityEngineInfo.similarityEngineType.value)), measure = Some(thriftlog.RequestTweetMeasure(candidate.predictionScore)) ) } ) } } private def buildRelatedTweetResponse(candidates: Seq[InitialCandidate]): RelatedTweetResponse = { val tweets = candidates.map { candidate => RelatedTweet( tweetId = candidate.tweetId, score = Some(candidate.getSimilarityScore), authorId = Some(candidate.tweetInfo.authorId) ) } RelatedTweetResponse(tweets) } private def buildRelatedVideoTweetResponse( candidates: Seq[InitialCandidate] ): RelatedVideoTweetResponse = { val tweets = candidates.map { candidate => RelatedVideoTweet( tweetId = candidate.tweetId, score = Some(candidate.getSimilarityScore) ) } RelatedVideoTweetResponse(tweets) } private def buildUtegTweetResponse( candidates: Seq[TweetWithScoreAndSocialProof] ): UtegTweetResponse = { val tweets = candidates.map { candidate => UtegTweet( tweetId = candidate.tweetId, score = candidate.score, socialProofByType = candidate.socialProofByType ) } UtegTweetResponse(tweets) } private def buildAdsResponse( candidates: Seq[RankedAdsCandidate] ): AdsResponse = { AdsResponse(ads = candidates.map { candidate => AdTweetRecommendation( tweetId = candidate.tweetId, score = candidate.predictionScore, lineItems = Some(candidate.lineItemInfo)) }) } private def cacheTweetRecommendationResults( request: CrMixerTweetRequest, response: Future[CrMixerTweetResponse] ): Unit = { val userId = request.clientContext.userId.getOrElse( throw new IllegalArgumentException( "userId must be present in getTweetRecommendations() Thrift clientContext")) if (decider.isAvailableForId(userId, DeciderConstants.getTweetRecommendationsCacheRate)) { response.map { crMixerTweetResponse => { ( request.product, request.clientContext.userId, crMixerTweetResponse.tweets.nonEmpty) match { case (t.Product.Home, Some(userId), true) => tweetRecommendationResultsStore.put((userId, crMixerTweetResponse)) case _ => Future.value(Unit) } } } } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/exception/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/exception/InvalidSANNConfigException.scala ================================================ package com.twitter.cr_mixer package exception case class InvalidSANNConfigException(msg: String) extends Exception(msg) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/featureswitch/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/javax/inject:javax.inject", "abdecider/src/main/scala", "configapi/configapi-abdecider", "configapi/configapi-core", "configapi/configapi-featureswitches:v2", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider", "cr-mixer/thrift/src/main/thrift:thrift-scala", "decider/src/main/scala", "discovery-common/src/main/scala/com/twitter/discovery/common/configapi", "featureswitches/featureswitches-core", "featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder", "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", "frigate/frigate-common:util", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/base", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/candidate", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/health", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/interests", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", "scribelib/marshallers/src/main/scala/com/twitter/scribelib/marshallers", "src/scala/com/twitter/simclusters_v2/common", "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", "src/thrift/com/twitter/frigate:frigate-common-thrift-scala", "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/featureswitch/CrMixerLoggingABDecider.scala ================================================ package com.twitter.cr_mixer package featureswitch import com.twitter.finagle.stats.StatsReceiver import com.twitter.abdecider.LoggingABDecider import com.twitter.abdecider.Recipient import com.twitter.abdecider.Bucket import com.twitter.frigate.common.util.StatsUtil import com.twitter.util.Local import scala.collection.concurrent.{Map => ConcurrentMap} /** * Wraps a LoggingABDecider, so all impressed buckets are recorded to a 'LocalContext' on a given request. * * Contexts (https://twitter.github.io/finagle/guide/Contexts.html) are Finagle's mechanism for * storing state/variables without having to pass these variables all around the request. * * In order for this class to be used the [[SetImpressedBucketsLocalContextFilter]] must be applied * at the beginning of the request, to initialize a concurrent map used to store impressed buckets. * * Whenever we get an a/b impression, the bucket information is logged to the concurrent hashmap. */ case class CrMixerLoggingABDecider( loggingAbDecider: LoggingABDecider, statsReceiver: StatsReceiver) extends LoggingABDecider { private val scopedStatsReceiver = statsReceiver.scope("cr_logging_ab_decider") override def impression( experimentName: String, recipient: Recipient ): Option[Bucket] = { StatsUtil.trackNonFutureBlockStats(scopedStatsReceiver.scope("log_impression")) { val maybeBuckets = loggingAbDecider.impression(experimentName, recipient) maybeBuckets.foreach { b => scopedStatsReceiver.counter("impressions").incr() CrMixerImpressedBuckets.recordImpressedBucket(b) } maybeBuckets } } override def track( experimentName: String, eventName: String, recipient: Recipient ): Unit = { loggingAbDecider.track(experimentName, eventName, recipient) } override def bucket( experimentName: String, recipient: Recipient ): Option[Bucket] = { loggingAbDecider.bucket(experimentName, recipient) } override def experiments: Seq[String] = loggingAbDecider.experiments override def experiment(experimentName: String) = loggingAbDecider.experiment(experimentName) } object CrMixerImpressedBuckets { private[featureswitch] val localImpressedBucketsMap = new Local[ConcurrentMap[Bucket, Boolean]] /** * Gets all impressed buckets for this request. **/ def getAllImpressedBuckets: Option[List[Bucket]] = { localImpressedBucketsMap.apply().map(_.map { case (k, _) => k }.toList) } private[featureswitch] def recordImpressedBucket(bucket: Bucket) = { localImpressedBucketsMap().foreach { m => m += bucket -> true } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/featureswitch/ParamsBuilder.scala ================================================ package com.twitter.cr_mixer.featureswitch import com.twitter.abdecider.LoggingABDecider import com.twitter.abdecider.UserRecipient import com.twitter.cr_mixer.{thriftscala => t} import com.twitter.core_workflows.user_model.thriftscala.UserState import com.twitter.discovery.common.configapi.FeatureContextBuilder import com.twitter.featureswitches.FSRecipient import com.twitter.featureswitches.UserAgent import com.twitter.featureswitches.{Recipient => FeatureSwitchRecipient} import com.twitter.finagle.stats.StatsReceiver import com.twitter.product_mixer.core.thriftscala.ClientContext import com.twitter.timelines.configapi.Config import com.twitter.timelines.configapi.FeatureValue import com.twitter.timelines.configapi.ForcedFeatureContext import com.twitter.timelines.configapi.OrElseFeatureContext import com.twitter.timelines.configapi.Params import com.twitter.timelines.configapi.RequestContext import com.twitter.timelines.configapi.abdecider.LoggingABDeciderExperimentContext import javax.inject.Inject import javax.inject.Singleton /** Singleton object for building [[Params]] to override */ @Singleton class ParamsBuilder @Inject() ( globalStats: StatsReceiver, abDecider: LoggingABDecider, featureContextBuilder: FeatureContextBuilder, config: Config) { private val stats = globalStats.scope("params") def buildFromClientContext( clientContext: ClientContext, product: t.Product, userState: UserState, userRoleOverride: Option[Set[String]] = None, featureOverrides: Map[String, FeatureValue] = Map.empty, ): Params = { clientContext.userId match { case Some(userId) => val userRecipient = buildFeatureSwitchRecipient( userId, userRoleOverride, clientContext, product, userState ) val featureContext = OrElseFeatureContext( ForcedFeatureContext(featureOverrides), featureContextBuilder( Some(userId), Some(userRecipient) )) config( requestContext = RequestContext( userId = Some(userId), experimentContext = LoggingABDeciderExperimentContext( abDecider, Some(UserRecipient(userId, Some(userId)))), featureContext = featureContext ), stats ) case None => val guestRecipient = buildFeatureSwitchRecipientWithGuestId(clientContext: ClientContext, product, userState) val featureContext = OrElseFeatureContext( ForcedFeatureContext(featureOverrides), featureContextBuilder( clientContext.userId, Some(guestRecipient) ) ) //ExperimentContext with GuestRecipient is not supported as there is no active use-cases yet in CrMixer config( requestContext = RequestContext( userId = clientContext.userId, featureContext = featureContext ), stats ) } } private def buildFeatureSwitchRecipientWithGuestId( clientContext: ClientContext, product: t.Product, userState: UserState ): FeatureSwitchRecipient = { val recipient = FSRecipient( userId = None, userRoles = None, deviceId = clientContext.deviceId, guestId = clientContext.guestId, languageCode = clientContext.languageCode, countryCode = clientContext.countryCode, userAgent = clientContext.userAgent.flatMap(UserAgent(_)), isVerified = None, isTwoffice = None, tooClient = None, highWaterMark = None ) recipient.withCustomFields( (ParamsBuilder.ProductCustomField, product.toString), (ParamsBuilder.UserStateCustomField, userState.toString) ) } private def buildFeatureSwitchRecipient( userId: Long, userRolesOverride: Option[Set[String]], clientContext: ClientContext, product: t.Product, userState: UserState ): FeatureSwitchRecipient = { val userRoles = userRolesOverride match { case Some(overrides) => Some(overrides) case _ => clientContext.userRoles.map(_.toSet) } val recipient = FSRecipient( userId = Some(userId), userRoles = userRoles, deviceId = clientContext.deviceId, guestId = clientContext.guestId, languageCode = clientContext.languageCode, countryCode = clientContext.countryCode, userAgent = clientContext.userAgent.flatMap(UserAgent(_)), isVerified = None, isTwoffice = None, tooClient = None, highWaterMark = None ) recipient.withCustomFields( (ParamsBuilder.ProductCustomField, product.toString), (ParamsBuilder.UserStateCustomField, userState.toString) ) } } object ParamsBuilder { private val ProductCustomField = "product_id" private val UserStateCustomField = "user_state" } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/featureswitch/SetImpressedBucketsLocalContextFilter.scala ================================================ package com.twitter.cr_mixer.featureswitch import com.twitter.finagle.Filter import javax.inject.Inject import javax.inject.Singleton import scala.collection.concurrent.TrieMap import com.twitter.abdecider.Bucket import com.twitter.finagle.Service @Singleton class SetImpressedBucketsLocalContextFilter @Inject() () extends Filter.TypeAgnostic { override def toFilter[Req, Rep]: Filter[Req, Rep, Req, Rep] = (request: Req, service: Service[Req, Rep]) => { val concurrentTrieMap = TrieMap .empty[Bucket, Boolean] // Trie map has no locks and O(1) inserts CrMixerImpressedBuckets.localImpressedBucketsMap.let(concurrentTrieMap) { service(request) } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/filter/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/storehaus:core", "3rdparty/jvm/javax/inject:javax.inject", "configapi/configapi-core", "content-recommender/thrift/src/main/thrift:thrift-scala", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", "cr-mixer/thrift/src/main/thrift:thrift-scala", "finagle/finagle-core/src/main", "frigate/frigate-common:util", "snowflake/src/main/scala/com/twitter/snowflake/id", "src/scala/com/twitter/simclusters_v2/common", "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/filter/FilterBase.scala ================================================ package com.twitter.cr_mixer.filter import com.twitter.cr_mixer.model.CandidateGeneratorQuery import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.util.Future trait FilterBase { def name: String type ConfigType def filter( candidates: Seq[Seq[InitialCandidate]], config: ConfigType ): Future[Seq[Seq[InitialCandidate]]] /** * Build the config params here. passing in param() into the filter is strongly discouraged * because param() can be slow when called many times */ def requestToConfig[CGQueryType <: CandidateGeneratorQuery](request: CGQueryType): ConfigType } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/filter/ImpressedTweetlistFilter.scala ================================================ package com.twitter.cr_mixer.filter import com.twitter.cr_mixer.model.CandidateGeneratorQuery import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.util.Future import javax.inject.Singleton @Singleton case class ImpressedTweetlistFilter() extends FilterBase { import ImpressedTweetlistFilter._ override val name: String = this.getClass.getCanonicalName override type ConfigType = FilterConfig /* Filtering removes some candidates based on configurable criteria. */ override def filter( candidates: Seq[Seq[InitialCandidate]], config: FilterConfig ): Future[Seq[Seq[InitialCandidate]]] = { // Remove candidates which match a source tweet, or which are passed in impressedTweetList val sourceTweetsMatch = candidates .flatMap { /*** * Within a Seq[Seq[InitialCandidate]], all candidates within a inner Seq * are guaranteed to have the same sourceInfo. Hence, we can pick .headOption * to represent the whole list when filtering by the internalId of the sourceInfoOpt. * But of course the similarityEngineInfo could be different. */ _.headOption.flatMap { candidate => candidate.candidateGenerationInfo.sourceInfoOpt.map(_.internalId) } }.collect { case InternalId.TweetId(id) => id } val impressedTweetList: Set[TweetId] = config.impressedTweetList ++ sourceTweetsMatch val filteredCandidateMap: Seq[Seq[InitialCandidate]] = candidates.map { _.filterNot { candidate => impressedTweetList.contains(candidate.tweetId) } } Future.value(filteredCandidateMap) } override def requestToConfig[CGQueryType <: CandidateGeneratorQuery]( request: CGQueryType ): FilterConfig = { FilterConfig(request.impressedTweetList) } } object ImpressedTweetlistFilter { case class FilterConfig(impressedTweetList: Set[TweetId]) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/filter/InNetworkFilter.scala ================================================ package com.twitter.cr_mixer.filter import com.twitter.cr_mixer.model.CandidateGeneratorQuery import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.UtegTweetCandidateGeneratorQuery import com.twitter.cr_mixer.param.UtegTweetGlobalParams import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.simclusters_v2.common.UserId import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import com.twitter.wtf.candidate.thriftscala.CandidateSeq import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton /*** * Filters in-network tweets */ @Singleton case class InNetworkFilter @Inject() ( @Named(ModuleNames.RealGraphInStore) realGraphStoreMh: ReadableStore[UserId, CandidateSeq], globalStats: StatsReceiver) extends FilterBase { override val name: String = this.getClass.getCanonicalName import InNetworkFilter._ override type ConfigType = FilterConfig private val stats: StatsReceiver = globalStats.scope(this.getClass.getCanonicalName) private val filterCandidatesStats = stats.scope("filter_candidates") override def filter( candidates: Seq[Seq[InitialCandidate]], filterConfig: FilterConfig, ): Future[Seq[Seq[InitialCandidate]]] = { StatsUtil.trackItemsStats(filterCandidatesStats) { filterCandidates(candidates, filterConfig) } } private def filterCandidates( candidates: Seq[Seq[InitialCandidate]], filterConfig: FilterConfig, ): Future[Seq[Seq[InitialCandidate]]] = { if (!filterConfig.enableInNetworkFilter) { Future.value(candidates) } else { filterConfig.userIdOpt match { case Some(userId) => realGraphStoreMh .get(userId).map(_.map(_.candidates.map(_.userId)).getOrElse(Seq.empty).toSet).map { realGraphInNetworkAuthorsSet => candidates.map(_.filterNot { candidate => realGraphInNetworkAuthorsSet.contains(candidate.tweetInfo.authorId) }) } case None => Future.value(candidates) } } } override def requestToConfig[CGQueryType <: CandidateGeneratorQuery]( request: CGQueryType ): FilterConfig = { request match { case UtegTweetCandidateGeneratorQuery(userId, _, _, _, _, params, _) => FilterConfig(Some(userId), params(UtegTweetGlobalParams.EnableInNetworkFilterParam)) case _ => FilterConfig(None, false) } } } object InNetworkFilter { case class FilterConfig( userIdOpt: Option[UserId], enableInNetworkFilter: Boolean) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/filter/PostRankFilterRunner.scala ================================================ package com.twitter.cr_mixer.filter import com.twitter.cr_mixer.model.CrCandidateGeneratorQuery import com.twitter.cr_mixer.model.RankedCandidate import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.finagle.stats.StatsReceiver import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton @Singleton case class PostRankFilterRunner @Inject() ( globalStats: StatsReceiver) { private val scopedStats = globalStats.scope(this.getClass.getCanonicalName) private val beforeCount = scopedStats.stat("candidate_count", "before") private val afterCount = scopedStats.stat("candidate_count", "after") def run( query: CrCandidateGeneratorQuery, candidates: Seq[RankedCandidate] ): Future[Seq[RankedCandidate]] = { beforeCount.add(candidates.size) Future( removeBadRecentNotificationCandidates(candidates) ).map { results => afterCount.add(results.size) results } } /** * Remove "bad" quality candidates generated by recent notifications * A candidate is bad when it is generated by a single RecentNotification * SourceKey. * e.x: * tweetA {recent notification1} -> bad * tweetB {recent notification1 recent notification2} -> good *tweetC {recent notification1 recent follow1} -> bad * SD-19397 */ private[filter] def removeBadRecentNotificationCandidates( candidates: Seq[RankedCandidate] ): Seq[RankedCandidate] = { candidates.filterNot { isBadQualityRecentNotificationCandidate } } private def isBadQualityRecentNotificationCandidate(candidate: RankedCandidate): Boolean = { candidate.potentialReasons.size == 1 && candidate.potentialReasons.head.sourceInfoOpt.nonEmpty && candidate.potentialReasons.head.sourceInfoOpt.get.sourceType == SourceType.NotificationClick } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/filter/PreRankFilterRunner.scala ================================================ package com.twitter.cr_mixer.filter import com.twitter.cr_mixer.model.CandidateGeneratorQuery import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.finagle.stats.StatsReceiver import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton @Singleton class PreRankFilterRunner @Inject() ( impressedTweetListFilter: ImpressedTweetlistFilter, tweetAgeFilter: TweetAgeFilter, videoTweetFilter: VideoTweetFilter, tweetReplyFilter: ReplyFilter, globalStats: StatsReceiver) { private val scopedStats = globalStats.scope(this.getClass.getCanonicalName) /*** * The order of the filters does not matter as long as we do not apply .take(N) truncation * across all filters. In other words, it is fine that we first do tweetAgeFilter, and then * we do impressedTweetListFilter, or the other way around. * Same idea applies to the signal based filter - it is ok that we apply signal based filters * before impressedTweetListFilter. * * We move all signal based filters before tweetAgeFilter and impressedTweetListFilter * as a set of early filters. */ val orderedFilters = Seq( tweetAgeFilter, impressedTweetListFilter, videoTweetFilter, tweetReplyFilter ) def runSequentialFilters[CGQueryType <: CandidateGeneratorQuery]( request: CGQueryType, candidates: Seq[Seq[InitialCandidate]], ): Future[Seq[Seq[InitialCandidate]]] = { PreRankFilterRunner.runSequentialFilters( request, candidates, orderedFilters, scopedStats ) } } object PreRankFilterRunner { private def recordCandidateStatsBeforeFilter( candidates: Seq[Seq[InitialCandidate]], statsReceiver: StatsReceiver ): Unit = { statsReceiver .counter("empty_sources", "before").incr( candidates.count { _.isEmpty } ) candidates.foreach { candidate => statsReceiver.counter("candidates", "before").incr(candidate.size) } } private def recordCandidateStatsAfterFilter( candidates: Seq[Seq[InitialCandidate]], statsReceiver: StatsReceiver ): Unit = { statsReceiver .counter("empty_sources", "after").incr( candidates.count { _.isEmpty } ) candidates.foreach { candidate => statsReceiver.counter("candidates", "after").incr(candidate.size) } } /* Helper function for running some candidates through a sequence of filters */ private[filter] def runSequentialFilters[CGQueryType <: CandidateGeneratorQuery]( request: CGQueryType, candidates: Seq[Seq[InitialCandidate]], filters: Seq[FilterBase], statsReceiver: StatsReceiver ): Future[Seq[Seq[InitialCandidate]]] = filters.foldLeft(Future.value(candidates)) { case (candsFut, filter) => candsFut.flatMap { cands => recordCandidateStatsBeforeFilter(cands, statsReceiver.scope(filter.name)) filter .filter(cands, filter.requestToConfig(request)) .map { filteredCands => recordCandidateStatsAfterFilter(filteredCands, statsReceiver.scope(filter.name)) filteredCands } } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/filter/ReplyFilter.scala ================================================ package com.twitter.cr_mixer.filter import com.twitter.cr_mixer.model.CandidateGeneratorQuery import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton /*** * Filters candidates that are replies */ @Singleton case class ReplyFilter @Inject() () extends FilterBase { override def name: String = this.getClass.getCanonicalName override type ConfigType = Boolean override def filter( candidates: Seq[Seq[InitialCandidate]], config: ConfigType ): Future[Seq[Seq[InitialCandidate]]] = { if (config) { Future.value( candidates.map { candidateSeq => candidateSeq.filterNot { candidate => candidate.tweetInfo.isReply.getOrElse(false) } } ) } else { Future.value(candidates) } } override def requestToConfig[CGQueryType <: CandidateGeneratorQuery]( query: CGQueryType ): ConfigType = { true } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/filter/RetweetFilter.scala ================================================ package com.twitter.cr_mixer.filter import com.twitter.cr_mixer.model.CandidateGeneratorQuery import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.param.UtegTweetGlobalParams import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton /*** * Filters candidates that are retweets */ @Singleton case class RetweetFilter @Inject() () extends FilterBase { override def name: String = this.getClass.getCanonicalName override type ConfigType = Boolean override def filter( candidates: Seq[Seq[InitialCandidate]], config: ConfigType ): Future[Seq[Seq[InitialCandidate]]] = { if (config) { Future.value( candidates.map { candidateSeq => candidateSeq.filterNot { candidate => candidate.tweetInfo.isRetweet.getOrElse(false) } } ) } else { Future.value(candidates) } } override def requestToConfig[CGQueryType <: CandidateGeneratorQuery]( query: CGQueryType ): ConfigType = { query.params(UtegTweetGlobalParams.EnableRetweetFilterParam) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/filter/TweetAgeFilter.scala ================================================ package com.twitter.cr_mixer.filter import com.twitter.cr_mixer.model.CandidateGeneratorQuery import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.param.GlobalParams import com.twitter.snowflake.id.SnowflakeId import com.twitter.util.Duration import com.twitter.util.Future import com.twitter.util.Time import javax.inject.Singleton import com.twitter.conversions.DurationOps._ @Singleton case class TweetAgeFilter() extends FilterBase { override val name: String = this.getClass.getCanonicalName override type ConfigType = Duration override def filter( candidates: Seq[Seq[InitialCandidate]], maxTweetAge: Duration ): Future[Seq[Seq[InitialCandidate]]] = { if (maxTweetAge >= 720.hours) { Future.value(candidates) } else { // Tweet IDs are approximately chronological (see http://go/snowflake), // so we are building the earliest tweet id once, // and pass that as the value to filter candidates for each CandidateGenerationModel. val earliestTweetId = SnowflakeId.firstIdFor(Time.now - maxTweetAge) Future.value(candidates.map(_.filter(_.tweetId >= earliestTweetId))) } } override def requestToConfig[CGQueryType <: CandidateGeneratorQuery]( query: CGQueryType ): Duration = { query.params(GlobalParams.MaxTweetAgeHoursParam) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/filter/TweetInfoHealthFilterBase.scala ================================================ package com.twitter.cr_mixer.filter import com.twitter.contentrecommender.thriftscala.TweetInfo import com.twitter.cr_mixer.model.CandidateGeneratorQuery import com.twitter.cr_mixer.model.CrCandidateGeneratorQuery import com.twitter.cr_mixer.model.HealthThreshold import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.util.Future import javax.inject.Singleton @Singleton trait TweetInfoHealthFilterBase extends FilterBase { override def name: String = this.getClass.getCanonicalName override type ConfigType = HealthThreshold.Enum.Value def thresholdToPropertyMap: Map[HealthThreshold.Enum.Value, TweetInfo => Option[Boolean]] def getFilterParamFn: CandidateGeneratorQuery => HealthThreshold.Enum.Value override def filter( candidates: Seq[Seq[InitialCandidate]], config: HealthThreshold.Enum.Value ): Future[Seq[Seq[InitialCandidate]]] = { Future.value(candidates.map { seq => seq.filter(p => thresholdToPropertyMap(config)(p.tweetInfo).getOrElse(true)) }) } /** * Build the config params here. passing in param() into the filter is strongly discouraged * because param() can be slow when called many times */ override def requestToConfig[CGQueryType <: CandidateGeneratorQuery]( query: CGQueryType ): HealthThreshold.Enum.Value = { query match { case q: CrCandidateGeneratorQuery => getFilterParamFn(q) case _ => HealthThreshold.Enum.Off } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/filter/UtegFilterRunner.scala ================================================ package com.twitter.cr_mixer.filter import com.twitter.cr_mixer.model.CandidateGeneratorQuery import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.finagle.stats.StatsReceiver import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton /*** * * Run filters sequentially for UTEG candidate generator. The structure is copied from PreRankFilterRunner. */ @Singleton class UtegFilterRunner @Inject() ( inNetworkFilter: InNetworkFilter, utegHealthFilter: UtegHealthFilter, retweetFilter: RetweetFilter, globalStats: StatsReceiver) { private val scopedStats = globalStats.scope(this.getClass.getCanonicalName) val orderedFilters: Seq[FilterBase] = Seq( inNetworkFilter, utegHealthFilter, retweetFilter ) def runSequentialFilters[CGQueryType <: CandidateGeneratorQuery]( request: CGQueryType, candidates: Seq[Seq[InitialCandidate]], ): Future[Seq[Seq[InitialCandidate]]] = { UtegFilterRunner.runSequentialFilters( request, candidates, orderedFilters, scopedStats ) } } object UtegFilterRunner { private def recordCandidateStatsBeforeFilter( candidates: Seq[Seq[InitialCandidate]], statsReceiver: StatsReceiver ): Unit = { statsReceiver .counter("empty_sources", "before").incr( candidates.count { _.isEmpty } ) candidates.foreach { candidate => statsReceiver.counter("candidates", "before").incr(candidate.size) } } private def recordCandidateStatsAfterFilter( candidates: Seq[Seq[InitialCandidate]], statsReceiver: StatsReceiver ): Unit = { statsReceiver .counter("empty_sources", "after").incr( candidates.count { _.isEmpty } ) candidates.foreach { candidate => statsReceiver.counter("candidates", "after").incr(candidate.size) } } /* Helper function for running some candidates through a sequence of filters */ private[filter] def runSequentialFilters[CGQueryType <: CandidateGeneratorQuery]( request: CGQueryType, candidates: Seq[Seq[InitialCandidate]], filters: Seq[FilterBase], statsReceiver: StatsReceiver ): Future[Seq[Seq[InitialCandidate]]] = filters.foldLeft(Future.value(candidates)) { case (candsFut, filter) => candsFut.flatMap { cands => recordCandidateStatsBeforeFilter(cands, statsReceiver.scope(filter.name)) filter .filter(cands, filter.requestToConfig(request)) .map { filteredCands => recordCandidateStatsAfterFilter(filteredCands, statsReceiver.scope(filter.name)) filteredCands } } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/filter/UtegHealthFilter.scala ================================================ package com.twitter.cr_mixer.filter import com.twitter.cr_mixer.model.CandidateGeneratorQuery import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.param.UtegTweetGlobalParams import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton /** * Remove unhealthy candidates * Currently Timeline Ranker applies a check on the following three scores: * - toxicityScore * - pBlockScore * - pReportedTweetScore * * Where isPassTweetHealthFilterStrict checks two additions scores with the same threshold: * - pSpammyTweetScore * - spammyTweetContentScore * * We've verified that both filters behave very similarly. */ @Singleton case class UtegHealthFilter @Inject() () extends FilterBase { override def name: String = this.getClass.getCanonicalName override type ConfigType = Boolean override def filter( candidates: Seq[Seq[InitialCandidate]], config: ConfigType ): Future[Seq[Seq[InitialCandidate]]] = { if (config) { Future.value( candidates.map { candidateSeq => candidateSeq.filter { candidate => candidate.tweetInfo.isPassTweetHealthFilterStrict.getOrElse(false) } } ) } else { Future.value(candidates) } } override def requestToConfig[CGQueryType <: CandidateGeneratorQuery]( query: CGQueryType ): ConfigType = { query.params(UtegTweetGlobalParams.EnableTLRHealthFilterParam) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/filter/VideoTweetFilter.scala ================================================ package com.twitter.cr_mixer.filter import com.twitter.cr_mixer.filter.VideoTweetFilter.FilterConfig import com.twitter.cr_mixer.model.CandidateGeneratorQuery import com.twitter.cr_mixer.model.CrCandidateGeneratorQuery import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.model.RelatedTweetCandidateGeneratorQuery import com.twitter.cr_mixer.model.RelatedVideoTweetCandidateGeneratorQuery import com.twitter.cr_mixer.param.VideoTweetFilterParams import com.twitter.util.Future import javax.inject.Singleton @Singleton case class VideoTweetFilter() extends FilterBase { override val name: String = this.getClass.getCanonicalName override type ConfigType = FilterConfig override def filter( candidates: Seq[Seq[InitialCandidate]], config: ConfigType ): Future[Seq[Seq[InitialCandidate]]] = { Future.value(candidates.map { _.flatMap { candidate => if (!config.enableVideoTweetFilter) { Some(candidate) } else { // if hasVideo is true, hasImage, hasGif should be false val hasVideo = checkTweetInfoAttribute(candidate.tweetInfo.hasVideo) val isHighMediaResolution = checkTweetInfoAttribute(candidate.tweetInfo.isHighMediaResolution) val isQuoteTweet = checkTweetInfoAttribute(candidate.tweetInfo.isQuoteTweet) val isReply = checkTweetInfoAttribute(candidate.tweetInfo.isReply) val hasMultipleMedia = checkTweetInfoAttribute(candidate.tweetInfo.hasMultipleMedia) val hasUrl = checkTweetInfoAttribute(candidate.tweetInfo.hasUrl) if (hasVideo && isHighMediaResolution && !isQuoteTweet && !isReply && !hasMultipleMedia && !hasUrl) { Some(candidate) } else { None } } } }) } def checkTweetInfoAttribute(attributeOpt: => Option[Boolean]): Boolean = { if (attributeOpt.isDefined) attributeOpt.get else { // takes Quoted Tweet (TweetInfo.isQuoteTweet) as an example, // if the attributeOpt is None, we by default say it is not a quoted tweet // similarly, if TweetInfo.hasVideo is a None, // we say it does not have video. false } } override def requestToConfig[CGQueryType <: CandidateGeneratorQuery]( query: CGQueryType ): FilterConfig = { val enableVideoTweetFilter = query match { case _: CrCandidateGeneratorQuery | _: RelatedTweetCandidateGeneratorQuery | _: RelatedVideoTweetCandidateGeneratorQuery => query.params(VideoTweetFilterParams.EnableVideoTweetFilterParam) case _ => false // e.g., GetRelatedTweets() } FilterConfig( enableVideoTweetFilter = enableVideoTweetFilter ) } } object VideoTweetFilter { // extend the filterConfig to add more flags if needed. // now they are hardcoded according to the prod setting case class FilterConfig( enableVideoTweetFilter: Boolean) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/AdsRecommendationsScribeLogger.scala ================================================ package com.twitter.cr_mixer.logging import com.twitter.cr_mixer.model.AdsCandidateGeneratorQuery import com.twitter.cr_mixer.model.InitialAdsCandidate import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.logging.ScribeLoggerUtils._ import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.thriftscala.AdsRecommendationTopLevelApiResult import com.twitter.cr_mixer.thriftscala.AdsRecommendationsResult import com.twitter.cr_mixer.thriftscala.AdsRequest import com.twitter.cr_mixer.thriftscala.AdsResponse import com.twitter.cr_mixer.thriftscala.FetchCandidatesResult import com.twitter.cr_mixer.thriftscala.GetAdsRecommendationsScribe import com.twitter.cr_mixer.thriftscala.PerformanceMetrics import com.twitter.cr_mixer.thriftscala.TweetCandidateWithMetadata import com.twitter.cr_mixer.util.CandidateGenerationKeyUtil import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.tracing.Trace import com.twitter.logging.Logger import com.twitter.simclusters_v2.common.UserId import com.twitter.util.Future import com.twitter.util.Stopwatch import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton case class AdsRecommendationsScribeLogger @Inject() ( @Named(ModuleNames.AdsRecommendationsLogger) adsRecommendationsScribeLogger: Logger, decider: CrMixerDecider, statsReceiver: StatsReceiver) { private val scopedStats = statsReceiver.scope(this.getClass.getCanonicalName) private val upperFunnelsStats = scopedStats.scope("UpperFunnels") private val topLevelApiStats = scopedStats.scope("TopLevelApi") /* * Scribe first step results after fetching initial ads candidate * */ def scribeInitialAdsCandidates( query: AdsCandidateGeneratorQuery, getResultFn: => Future[Seq[Seq[InitialAdsCandidate]]], enableScribe: Boolean // controlled by feature switch so that we can scribe for certain DDG ): Future[Seq[Seq[InitialAdsCandidate]]] = { val scribeMetadata = ScribeMetadata.from(query) val timer = Stopwatch.start() getResultFn.onSuccess { input => val latencyMs = timer().inMilliseconds val result = convertFetchCandidatesResult(input, scribeMetadata.userId) val traceId = Trace.id.traceId.toLong val scribeMsg = buildScribeMessage(result, scribeMetadata, latencyMs, traceId) if (enableScribe && decider.isAvailableForId( scribeMetadata.userId, DeciderConstants.adsRecommendationsPerExperimentScribeRate)) { upperFunnelsStats.counter(scribeMetadata.product.originalName).incr() scribeResult(scribeMsg) } } } /* * Scribe top level API results * */ def scribeGetAdsRecommendations( request: AdsRequest, startTime: Long, scribeMetadata: ScribeMetadata, getResultFn: => Future[AdsResponse], enableScribe: Boolean ): Future[AdsResponse] = { val timer = Stopwatch.start() getResultFn.onSuccess { response => val latencyMs = timer().inMilliseconds val result = AdsRecommendationsResult.AdsRecommendationTopLevelApiResult( AdsRecommendationTopLevelApiResult( timestamp = startTime, request = request, response = response )) val traceId = Trace.id.traceId.toLong val scribeMsg = buildScribeMessage(result, scribeMetadata, latencyMs, traceId) if (enableScribe && decider.isAvailableForId( scribeMetadata.userId, DeciderConstants.adsRecommendationsPerExperimentScribeRate)) { topLevelApiStats.counter(scribeMetadata.product.originalName).incr() scribeResult(scribeMsg) } } } private def convertFetchCandidatesResult( candidatesSeq: Seq[Seq[InitialAdsCandidate]], requestUserId: UserId ): AdsRecommendationsResult = { val tweetCandidatesWithMetadata = candidatesSeq.flatMap { candidates => candidates.map { candidate => TweetCandidateWithMetadata( tweetId = candidate.tweetId, candidateGenerationKey = Some( CandidateGenerationKeyUtil.toThrift(candidate.candidateGenerationInfo, requestUserId)), score = Some(candidate.getSimilarityScore), numCandidateGenerationKeys = None // not populated yet ) } } AdsRecommendationsResult.FetchCandidatesResult( FetchCandidatesResult(Some(tweetCandidatesWithMetadata))) } private def buildScribeMessage( result: AdsRecommendationsResult, scribeMetadata: ScribeMetadata, latencyMs: Long, traceId: Long ): GetAdsRecommendationsScribe = { GetAdsRecommendationsScribe( uuid = scribeMetadata.requestUUID, userId = scribeMetadata.userId, result = result, traceId = Some(traceId), performanceMetrics = Some(PerformanceMetrics(Some(latencyMs))), impressedBuckets = getImpressedBuckets(scopedStats) ) } private def scribeResult( scribeMsg: GetAdsRecommendationsScribe ): Unit = { publish( logger = adsRecommendationsScribeLogger, codec = GetAdsRecommendationsScribe, message = scribeMsg) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/javax/inject:javax.inject", "abdecider/src/main/scala", "content-recommender/thrift/src/main/thrift:content-recommender-common-scala", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/featureswitch", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util", "cr-mixer/thrift/src/main/thrift:thrift-scala", "decider/src/main/scala", "featureswitches/featureswitches-core/src/main/scala:experimentation-settings", "finagle/finagle-core/src/main", "frigate/frigate-common:base", "frigate/frigate-common:util", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/base", "kafka/finagle-kafka/finatra-kafka/src/main/scala", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", "scribelib/marshallers/src/main/scala/com/twitter/scribelib/marshallers", "scribelib/validators/src/main/scala/com/twitter/scribelib/validators", "scrooge/scrooge-serializer/src/main/scala", "src/scala/com/twitter/simclusters_v2/common", "src/thrift/com/twitter/ml/api:data-scala", "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", "timelines/src/main/scala/com/twitter/timelines/clientevent", "util-internal/scribe/src/main/scala/com/twitter/logging", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/CrMixerScribeLogger.scala ================================================ package com.twitter.cr_mixer.logging import com.google.common.base.CaseFormat import com.twitter.abdecider.ScribingABDeciderUtil import com.twitter.scribelib.marshallers.ClientDataProvider import com.twitter.scribelib.marshallers.ScribeSerialization import com.twitter.timelines.clientevent.MinimalClientDataProvider import com.twitter.cr_mixer.model.BlendedCandidate import com.twitter.cr_mixer.model.CrCandidateGeneratorQuery import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.model.RankedCandidate import com.twitter.cr_mixer.logging.ScribeLoggerUtils._ import com.twitter.cr_mixer.model.GraphSourceInfo import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.scribe.ScribeCategories import com.twitter.cr_mixer.thriftscala.CrMixerTweetRequest import com.twitter.cr_mixer.thriftscala.CrMixerTweetResponse import com.twitter.cr_mixer.thriftscala.FetchCandidatesResult import com.twitter.cr_mixer.thriftscala.FetchSignalSourcesResult import com.twitter.cr_mixer.thriftscala.GetTweetsRecommendationsScribe import com.twitter.cr_mixer.thriftscala.InterleaveResult import com.twitter.cr_mixer.thriftscala.PerformanceMetrics import com.twitter.cr_mixer.thriftscala.PreRankFilterResult import com.twitter.cr_mixer.thriftscala.Product import com.twitter.cr_mixer.thriftscala.RankResult import com.twitter.cr_mixer.thriftscala.Result import com.twitter.cr_mixer.thriftscala.SourceSignal import com.twitter.cr_mixer.thriftscala.TopLevelApiResult import com.twitter.cr_mixer.thriftscala.TweetCandidateWithMetadata import com.twitter.cr_mixer.thriftscala.VITTweetCandidateScribe import com.twitter.cr_mixer.thriftscala.VITTweetCandidatesScribe import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.util.CandidateGenerationKeyUtil import com.twitter.cr_mixer.util.MetricTagUtil import com.twitter.decider.SimpleRecipient import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.tracing.Trace import com.twitter.finatra.kafka.producers.KafkaProducerBase import com.twitter.logging.Logger import com.twitter.simclusters_v2.common.UserId import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.util.Future import com.twitter.util.Stopwatch import com.twitter.util.Time import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.util.Random @Singleton case class CrMixerScribeLogger @Inject() ( decider: CrMixerDecider, statsReceiver: StatsReceiver, @Named(ModuleNames.TweetRecsLogger) tweetRecsScribeLogger: Logger, @Named(ModuleNames.BlueVerifiedTweetRecsLogger) blueVerifiedTweetRecsScribeLogger: Logger, @Named(ModuleNames.TopLevelApiDdgMetricsLogger) ddgMetricsLogger: Logger, kafkaProducer: KafkaProducerBase[String, GetTweetsRecommendationsScribe]) { import CrMixerScribeLogger._ private val scopedStats = statsReceiver.scope("CrMixerScribeLogger") private val topLevelApiStats = scopedStats.scope("TopLevelApi") private val upperFunnelsStats = scopedStats.scope("UpperFunnels") private val kafkaMessagesStats = scopedStats.scope("KafkaMessages") private val topLevelApiDdgMetricsStats = scopedStats.scope("TopLevelApiDdgMetrics") private val blueVerifiedTweetCandidatesStats = scopedStats.scope("BlueVerifiedTweetCandidates") private val serialization = new ScribeSerialization {} def scribeSignalSources( query: CrCandidateGeneratorQuery, getResultFn: => Future[(Set[SourceInfo], Map[String, Option[GraphSourceInfo]])] ): Future[(Set[SourceInfo], Map[String, Option[GraphSourceInfo]])] = { scribeResultsAndPerformanceMetrics( ScribeMetadata.from(query), getResultFn, convertToResultFn = convertFetchSignalSourcesResult ) } def scribeInitialCandidates( query: CrCandidateGeneratorQuery, getResultFn: => Future[Seq[Seq[InitialCandidate]]] ): Future[Seq[Seq[InitialCandidate]]] = { scribeResultsAndPerformanceMetrics( ScribeMetadata.from(query), getResultFn, convertToResultFn = convertFetchCandidatesResult ) } def scribePreRankFilterCandidates( query: CrCandidateGeneratorQuery, getResultFn: => Future[Seq[Seq[InitialCandidate]]] ): Future[Seq[Seq[InitialCandidate]]] = { scribeResultsAndPerformanceMetrics( ScribeMetadata.from(query), getResultFn, convertToResultFn = convertPreRankFilterResult ) } def scribeInterleaveCandidates( query: CrCandidateGeneratorQuery, getResultFn: => Future[Seq[BlendedCandidate]] ): Future[Seq[BlendedCandidate]] = { scribeResultsAndPerformanceMetrics( ScribeMetadata.from(query), getResultFn, convertToResultFn = convertInterleaveResult, enableKafkaScribe = true ) } def scribeRankedCandidates( query: CrCandidateGeneratorQuery, getResultFn: => Future[Seq[RankedCandidate]] ): Future[Seq[RankedCandidate]] = { scribeResultsAndPerformanceMetrics( ScribeMetadata.from(query), getResultFn, convertToResultFn = convertRankResult ) } /** * Scribe Top Level API Request / Response and performance metrics * for the getTweetRecommendations() endpoint. */ def scribeGetTweetRecommendations( request: CrMixerTweetRequest, startTime: Long, scribeMetadata: ScribeMetadata, getResultFn: => Future[CrMixerTweetResponse] ): Future[CrMixerTweetResponse] = { val timer = Stopwatch.start() getResultFn.onSuccess { response => val latencyMs = timer().inMilliseconds val result = convertTopLevelAPIResult(request, response, startTime) val traceId = Trace.id.traceId.toLong val scribeMsg = buildScribeMessage(result, scribeMetadata, latencyMs, traceId) // We use upperFunnelPerStepScribeRate to cover TopLevelApi scribe logs if (decider.isAvailableForId( scribeMetadata.userId, DeciderConstants.upperFunnelPerStepScribeRate)) { topLevelApiStats.counter(scribeMetadata.product.originalName).incr() scribeResult(scribeMsg) } if (decider.isAvailableForId( scribeMetadata.userId, DeciderConstants.topLevelApiDdgMetricsScribeRate)) { topLevelApiDdgMetricsStats.counter(scribeMetadata.product.originalName).incr() val topLevelDdgMetricsMetadata = TopLevelDdgMetricsMetadata.from(request) publishTopLevelDdgMetrics( logger = ddgMetricsLogger, topLevelDdgMetricsMetadata = topLevelDdgMetricsMetadata, latencyMs = latencyMs, candidateSize = response.tweets.length) } } } /** * Scribe all of the Blue Verified tweets that are candidates from cr-mixer * from the getTweetRecommendations() endpoint for stats tracking/debugging purposes. */ def scribeGetTweetRecommendationsForBlueVerified( scribeMetadata: ScribeMetadata, getResultFn: => Future[Seq[RankedCandidate]] ): Future[Seq[RankedCandidate]] = { getResultFn.onSuccess { rankedCandidates => if (decider.isAvailable(DeciderConstants.enableScribeForBlueVerifiedTweetCandidates)) { blueVerifiedTweetCandidatesStats.counter("process_request").incr() val blueVerifiedTweetCandidates = rankedCandidates.filter { tweet => tweet.tweetInfo.hasBlueVerifiedAnnotation.contains(true) } val impressedBuckets = getImpressedBuckets(blueVerifiedTweetCandidatesStats).getOrElse(Nil) val blueVerifiedCandidateScribes = blueVerifiedTweetCandidates.map { candidate => blueVerifiedTweetCandidatesStats .scope(scribeMetadata.product.name).counter( candidate.tweetInfo.authorId.toString).incr() VITTweetCandidateScribe( tweetId = candidate.tweetId, authorId = candidate.tweetInfo.authorId, score = candidate.predictionScore, metricTags = MetricTagUtil.buildMetricTags(candidate) ) } val blueVerifiedScribe = VITTweetCandidatesScribe( uuid = scribeMetadata.requestUUID, userId = scribeMetadata.userId, candidates = blueVerifiedCandidateScribes, product = scribeMetadata.product, impressedBuckets = impressedBuckets ) publish( logger = blueVerifiedTweetRecsScribeLogger, codec = VITTweetCandidatesScribe, message = blueVerifiedScribe) } } } /** * Scribe Per-step intermediate results and performance metrics * for each step: fetch signals, fetch candidates, filters, ranker, etc */ private[logging] def scribeResultsAndPerformanceMetrics[T]( scribeMetadata: ScribeMetadata, getResultFn: => Future[T], convertToResultFn: (T, UserId) => Result, enableKafkaScribe: Boolean = false ): Future[T] = { val timer = Stopwatch.start() getResultFn.onSuccess { input => val latencyMs = timer().inMilliseconds val result = convertToResultFn(input, scribeMetadata.userId) val traceId = Trace.id.traceId.toLong val scribeMsg = buildScribeMessage(result, scribeMetadata, latencyMs, traceId) if (decider.isAvailableForId( scribeMetadata.userId, DeciderConstants.upperFunnelPerStepScribeRate)) { upperFunnelsStats.counter(scribeMetadata.product.originalName).incr() scribeResult(scribeMsg) } // forks the scribe as a Kafka message for async feature hydration if (enableKafkaScribe && shouldScribeKafkaMessage( scribeMetadata.userId, scribeMetadata.product)) { kafkaMessagesStats.counter(scribeMetadata.product.originalName).incr() val batchedKafkaMessages = downsampleKafkaMessage(scribeMsg) batchedKafkaMessages.foreach { kafkaMessage => kafkaProducer.send( topic = ScribeCategories.TweetsRecs.scribeCategory, key = traceId.toString, value = kafkaMessage, timestamp = Time.now.inMilliseconds ) } } } } private def convertTopLevelAPIResult( request: CrMixerTweetRequest, response: CrMixerTweetResponse, startTime: Long ): Result = { Result.TopLevelApiResult( TopLevelApiResult( timestamp = startTime, request = request, response = response )) } private def convertFetchSignalSourcesResult( sourceInfoSetTuple: (Set[SourceInfo], Map[String, Option[GraphSourceInfo]]), requestUserId: UserId ): Result = { val sourceSignals = sourceInfoSetTuple._1.map { sourceInfo => SourceSignal(id = Some(sourceInfo.internalId)) } // For source graphs, we pass in requestUserId as a placeholder val sourceGraphs = sourceInfoSetTuple._2.map { case (_, _) => SourceSignal(id = Some(InternalId.UserId(requestUserId))) } Result.FetchSignalSourcesResult( FetchSignalSourcesResult( signals = Some(sourceSignals ++ sourceGraphs) )) } private def convertFetchCandidatesResult( candidatesSeq: Seq[Seq[InitialCandidate]], requestUserId: UserId ): Result = { val tweetCandidatesWithMetadata = candidatesSeq.flatMap { candidates => candidates.map { candidate => TweetCandidateWithMetadata( tweetId = candidate.tweetId, candidateGenerationKey = Some( CandidateGenerationKeyUtil.toThrift(candidate.candidateGenerationInfo, requestUserId)), score = Some(candidate.getSimilarityScore), numCandidateGenerationKeys = None // not populated yet ) } } Result.FetchCandidatesResult(FetchCandidatesResult(Some(tweetCandidatesWithMetadata))) } private def convertPreRankFilterResult( candidatesSeq: Seq[Seq[InitialCandidate]], requestUserId: UserId ): Result = { val tweetCandidatesWithMetadata = candidatesSeq.flatMap { candidates => candidates.map { candidate => TweetCandidateWithMetadata( tweetId = candidate.tweetId, candidateGenerationKey = Some( CandidateGenerationKeyUtil.toThrift(candidate.candidateGenerationInfo, requestUserId)), score = Some(candidate.getSimilarityScore), numCandidateGenerationKeys = None // not populated yet ) } } Result.PreRankFilterResult(PreRankFilterResult(Some(tweetCandidatesWithMetadata))) } // We take InterleaveResult for Unconstrained dataset ML ranker training private def convertInterleaveResult( blendedCandidates: Seq[BlendedCandidate], requestUserId: UserId ): Result = { val tweetCandidatesWithMetadata = blendedCandidates.map { blendedCandidate => val candidateGenerationKey = CandidateGenerationKeyUtil.toThrift(blendedCandidate.reasonChosen, requestUserId) TweetCandidateWithMetadata( tweetId = blendedCandidate.tweetId, candidateGenerationKey = Some(candidateGenerationKey), authorId = Some(blendedCandidate.tweetInfo.authorId), // for ML pipeline training score = Some(blendedCandidate.getSimilarityScore), numCandidateGenerationKeys = Some(blendedCandidate.potentialReasons.size) ) // hydrate fields for light ranking training data } Result.InterleaveResult(InterleaveResult(Some(tweetCandidatesWithMetadata))) } private def convertRankResult( rankedCandidates: Seq[RankedCandidate], requestUserId: UserId ): Result = { val tweetCandidatesWithMetadata = rankedCandidates.map { rankedCandidate => val candidateGenerationKey = CandidateGenerationKeyUtil.toThrift(rankedCandidate.reasonChosen, requestUserId) TweetCandidateWithMetadata( tweetId = rankedCandidate.tweetId, candidateGenerationKey = Some(candidateGenerationKey), score = Some(rankedCandidate.getSimilarityScore), numCandidateGenerationKeys = Some(rankedCandidate.potentialReasons.size) ) } Result.RankResult(RankResult(Some(tweetCandidatesWithMetadata))) } private def buildScribeMessage( result: Result, scribeMetadata: ScribeMetadata, latencyMs: Long, traceId: Long ): GetTweetsRecommendationsScribe = { GetTweetsRecommendationsScribe( uuid = scribeMetadata.requestUUID, userId = scribeMetadata.userId, result = result, traceId = Some(traceId), performanceMetrics = Some(PerformanceMetrics(Some(latencyMs))), impressedBuckets = getImpressedBuckets(scopedStats) ) } private def scribeResult( scribeMsg: GetTweetsRecommendationsScribe ): Unit = { publish( logger = tweetRecsScribeLogger, codec = GetTweetsRecommendationsScribe, message = scribeMsg) } /** * Gate for producing messages to Kafka for async feature hydration */ private def shouldScribeKafkaMessage( userId: UserId, product: Product ): Boolean = { val isEligibleUser = decider.isAvailable( DeciderConstants.kafkaMessageScribeSampleRate, Some(SimpleRecipient(userId))) val isHomeProduct = (product == Product.Home) isEligibleUser && isHomeProduct } /** * Due to size limits of Strato (see SD-19028), each Kafka message must be downsampled */ private[logging] def downsampleKafkaMessage( scribeMsg: GetTweetsRecommendationsScribe ): Seq[GetTweetsRecommendationsScribe] = { val sampledResultSeq: Seq[Result] = scribeMsg.result match { case Result.InterleaveResult(interleaveResult) => val sampledTweetsSeq = interleaveResult.tweets .map { tweets => Random .shuffle(tweets).take(KafkaMaxTweetsPerMessage) .grouped(BatchSize).toSeq }.getOrElse(Seq.empty) sampledTweetsSeq.map { sampledTweets => Result.InterleaveResult(InterleaveResult(Some(sampledTweets))) } // if it's an unrecognized type, err on the side of sending no candidates case _ => kafkaMessagesStats.counter("InvalidKafkaMessageResultType").incr() Seq(Result.InterleaveResult(InterleaveResult(None))) } sampledResultSeq.map { sampledResult => GetTweetsRecommendationsScribe( uuid = scribeMsg.uuid, userId = scribeMsg.userId, result = sampledResult, traceId = scribeMsg.traceId, performanceMetrics = None, impressedBuckets = None ) } } /** * Handles client_event serialization to log data into DDG metrics */ private[logging] def publishTopLevelDdgMetrics( logger: Logger, topLevelDdgMetricsMetadata: TopLevelDdgMetricsMetadata, candidateSize: Long, latencyMs: Long, ): Unit = { val data = Map[Any, Any]( "latency_ms" -> latencyMs, "event_value" -> candidateSize ) val label: (String, String) = ("tweetrec", "") val namespace = getNamespace(topLevelDdgMetricsMetadata, label) + ("action" -> "candidates") val message = serialization .serializeClientEvent(namespace, getClientData(topLevelDdgMetricsMetadata), data) logger.info(message) } private def getClientData( topLevelDdgMetricsMetadata: TopLevelDdgMetricsMetadata ): ClientDataProvider = MinimalClientDataProvider( userId = topLevelDdgMetricsMetadata.userId, guestId = None, clientApplicationId = topLevelDdgMetricsMetadata.clientApplicationId, countryCode = topLevelDdgMetricsMetadata.countryCode ) private def getNamespace( topLevelDdgMetricsMetadata: TopLevelDdgMetricsMetadata, label: (String, String) ): Map[String, String] = { val productName = CaseFormat.UPPER_CAMEL .to(CaseFormat.LOWER_UNDERSCORE, topLevelDdgMetricsMetadata.product.originalName) Map( "client" -> ScribingABDeciderUtil.clientForAppId( topLevelDdgMetricsMetadata.clientApplicationId), "page" -> "cr-mixer", "section" -> productName, "component" -> label._1, "element" -> label._2 ) } } object CrMixerScribeLogger { val KafkaMaxTweetsPerMessage: Int = 200 val BatchSize: Int = 20 } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/RelatedTweetScribeLogger.scala ================================================ package com.twitter.cr_mixer.logging import com.twitter.cr_mixer.model.RelatedTweetCandidateGeneratorQuery import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.logging.ScribeLoggerUtils._ import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.thriftscala.FetchCandidatesResult import com.twitter.cr_mixer.thriftscala.GetRelatedTweetsScribe import com.twitter.cr_mixer.thriftscala.PerformanceMetrics import com.twitter.cr_mixer.thriftscala.PreRankFilterResult import com.twitter.cr_mixer.thriftscala.RelatedTweetRequest import com.twitter.cr_mixer.thriftscala.RelatedTweetResponse import com.twitter.cr_mixer.thriftscala.RelatedTweetResult import com.twitter.cr_mixer.thriftscala.RelatedTweetTopLevelApiResult import com.twitter.cr_mixer.thriftscala.TweetCandidateWithMetadata import com.twitter.cr_mixer.util.CandidateGenerationKeyUtil import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.tracing.Trace import com.twitter.logging.Logger import com.twitter.simclusters_v2.common.UserId import com.twitter.util.Future import com.twitter.util.Stopwatch import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton case class RelatedTweetScribeLogger @Inject() ( decider: CrMixerDecider, statsReceiver: StatsReceiver, @Named(ModuleNames.RelatedTweetsLogger) relatedTweetsScribeLogger: Logger) { private val scopedStats = statsReceiver.scope("RelatedTweetsScribeLogger") private val topLevelApiStats = scopedStats.scope("TopLevelApi") private val topLevelApiNoUserIdStats = scopedStats.scope("TopLevelApiNoUserId") private val upperFunnelsStats = scopedStats.scope("UpperFunnels") private val upperFunnelsNoUserIdStats = scopedStats.scope("UpperFunnelsNoUserId") def scribeInitialCandidates( query: RelatedTweetCandidateGeneratorQuery, getResultFn: => Future[Seq[Seq[InitialCandidate]]] ): Future[Seq[Seq[InitialCandidate]]] = { scribeResultsAndPerformanceMetrics( RelatedTweetScribeMetadata.from(query), getResultFn, convertToResultFn = convertFetchCandidatesResult ) } def scribePreRankFilterCandidates( query: RelatedTweetCandidateGeneratorQuery, getResultFn: => Future[Seq[Seq[InitialCandidate]]] ): Future[Seq[Seq[InitialCandidate]]] = { scribeResultsAndPerformanceMetrics( RelatedTweetScribeMetadata.from(query), getResultFn, convertToResultFn = convertPreRankFilterResult ) } /** * Scribe Top Level API Request / Response and performance metrics * for the getRelatedTweets endpoint. */ def scribeGetRelatedTweets( request: RelatedTweetRequest, startTime: Long, relatedTweetScribeMetadata: RelatedTweetScribeMetadata, getResultFn: => Future[RelatedTweetResponse] ): Future[RelatedTweetResponse] = { val timer = Stopwatch.start() getResultFn.onSuccess { response => relatedTweetScribeMetadata.clientContext.userId match { case Some(userId) => if (decider.isAvailableForId(userId, DeciderConstants.upperFunnelPerStepScribeRate)) { topLevelApiStats.counter(relatedTweetScribeMetadata.product.originalName).incr() val latencyMs = timer().inMilliseconds val result = convertTopLevelAPIResult(request, response, startTime) val traceId = Trace.id.traceId.toLong val scribeMsg = buildScribeMessage(result, relatedTweetScribeMetadata, latencyMs, traceId) scribeResult(scribeMsg) } case _ => topLevelApiNoUserIdStats.counter(relatedTweetScribeMetadata.product.originalName).incr() } } } /** * Scribe Per-step intermediate results and performance metrics * for each step: fetch candidates, filters. */ private def scribeResultsAndPerformanceMetrics[T]( relatedTweetScribeMetadata: RelatedTweetScribeMetadata, getResultFn: => Future[T], convertToResultFn: (T, UserId) => RelatedTweetResult ): Future[T] = { val timer = Stopwatch.start() getResultFn.onSuccess { input => relatedTweetScribeMetadata.clientContext.userId match { case Some(userId) => if (decider.isAvailableForId(userId, DeciderConstants.upperFunnelPerStepScribeRate)) { upperFunnelsStats.counter(relatedTweetScribeMetadata.product.originalName).incr() val latencyMs = timer().inMilliseconds val result = convertToResultFn(input, userId) val traceId = Trace.id.traceId.toLong val scribeMsg = buildScribeMessage(result, relatedTweetScribeMetadata, latencyMs, traceId) scribeResult(scribeMsg) } case _ => upperFunnelsNoUserIdStats.counter(relatedTweetScribeMetadata.product.originalName).incr() } } } private def convertTopLevelAPIResult( request: RelatedTweetRequest, response: RelatedTweetResponse, startTime: Long ): RelatedTweetResult = { RelatedTweetResult.RelatedTweetTopLevelApiResult( RelatedTweetTopLevelApiResult( timestamp = startTime, request = request, response = response )) } private def convertFetchCandidatesResult( candidatesSeq: Seq[Seq[InitialCandidate]], requestUserId: UserId ): RelatedTweetResult = { val tweetCandidatesWithMetadata = candidatesSeq.flatMap { candidates => candidates.map { candidate => TweetCandidateWithMetadata( tweetId = candidate.tweetId, candidateGenerationKey = None ) // do not hydrate candidateGenerationKey to save cost } } RelatedTweetResult.FetchCandidatesResult( FetchCandidatesResult(Some(tweetCandidatesWithMetadata))) } private def convertPreRankFilterResult( candidatesSeq: Seq[Seq[InitialCandidate]], requestUserId: UserId ): RelatedTweetResult = { val tweetCandidatesWithMetadata = candidatesSeq.flatMap { candidates => candidates.map { candidate => val candidateGenerationKey = CandidateGenerationKeyUtil.toThrift(candidate.candidateGenerationInfo, requestUserId) TweetCandidateWithMetadata( tweetId = candidate.tweetId, candidateGenerationKey = Some(candidateGenerationKey), authorId = Some(candidate.tweetInfo.authorId), score = Some(candidate.getSimilarityScore), numCandidateGenerationKeys = None ) } } RelatedTweetResult.PreRankFilterResult(PreRankFilterResult(Some(tweetCandidatesWithMetadata))) } private def buildScribeMessage( relatedTweetResult: RelatedTweetResult, relatedTweetScribeMetadata: RelatedTweetScribeMetadata, latencyMs: Long, traceId: Long ): GetRelatedTweetsScribe = { GetRelatedTweetsScribe( uuid = relatedTweetScribeMetadata.requestUUID, internalId = relatedTweetScribeMetadata.internalId, relatedTweetResult = relatedTweetResult, requesterId = relatedTweetScribeMetadata.clientContext.userId, guestId = relatedTweetScribeMetadata.clientContext.guestId, traceId = Some(traceId), performanceMetrics = Some(PerformanceMetrics(Some(latencyMs))), impressedBuckets = getImpressedBuckets(scopedStats) ) } private def scribeResult( scribeMsg: GetRelatedTweetsScribe ): Unit = { publish(logger = relatedTweetsScribeLogger, codec = GetRelatedTweetsScribe, message = scribeMsg) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/ScribeLoggerUtils.scala ================================================ package com.twitter.cr_mixer.logging import com.twitter.cr_mixer.featureswitch.CrMixerImpressedBuckets import com.twitter.cr_mixer.thriftscala.ImpressesedBucketInfo import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.logging.Logger import com.twitter.scrooge.BinaryThriftStructSerializer import com.twitter.scrooge.ThriftStruct import com.twitter.scrooge.ThriftStructCodec object ScribeLoggerUtils { /** * Handles base64-encoding, serialization, and publish. */ private[logging] def publish[T <: ThriftStruct]( logger: Logger, codec: ThriftStructCodec[T], message: T ): Unit = { logger.info(BinaryThriftStructSerializer(codec).toString(message)) } private[logging] def getImpressedBuckets( scopedStats: StatsReceiver ): Option[List[ImpressesedBucketInfo]] = { StatsUtil.trackNonFutureBlockStats(scopedStats.scope("getImpressedBuckets")) { CrMixerImpressedBuckets.getAllImpressedBuckets.map { listBuckets => val listBucketsSet = listBuckets.toSet scopedStats.stat("impressed_buckets").add(listBucketsSet.size) listBucketsSet.map { bucket => ImpressesedBucketInfo( experimentId = bucket.experiment.settings.experimentId.getOrElse(-1L), bucketName = bucket.name, version = bucket.experiment.settings.version, ) }.toList } } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/ScribeMetadata.scala ================================================ package com.twitter.cr_mixer.logging import com.twitter.cr_mixer.model.AdsCandidateGeneratorQuery import com.twitter.cr_mixer.model.CrCandidateGeneratorQuery import com.twitter.cr_mixer.model.RelatedTweetCandidateGeneratorQuery import com.twitter.cr_mixer.model.UtegTweetCandidateGeneratorQuery import com.twitter.cr_mixer.thriftscala.Product import com.twitter.product_mixer.core.thriftscala.ClientContext import com.twitter.simclusters_v2.common.UserId import com.twitter.simclusters_v2.thriftscala.InternalId case class ScribeMetadata( requestUUID: Long, userId: UserId, product: Product) object ScribeMetadata { def from(query: CrCandidateGeneratorQuery): ScribeMetadata = { ScribeMetadata(query.requestUUID, query.userId, query.product) } def from(query: UtegTweetCandidateGeneratorQuery): ScribeMetadata = { ScribeMetadata(query.requestUUID, query.userId, query.product) } def from(query: AdsCandidateGeneratorQuery): ScribeMetadata = { ScribeMetadata(query.requestUUID, query.userId, query.product) } } case class RelatedTweetScribeMetadata( requestUUID: Long, internalId: InternalId, clientContext: ClientContext, product: Product) object RelatedTweetScribeMetadata { def from(query: RelatedTweetCandidateGeneratorQuery): RelatedTweetScribeMetadata = { RelatedTweetScribeMetadata( query.requestUUID, query.internalId, query.clientContext, query.product) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/TopLevelDdgMetricsMetadata.scala ================================================ package com.twitter.cr_mixer package logging import com.twitter.cr_mixer.thriftscala.CrMixerTweetRequest import com.twitter.cr_mixer.thriftscala.Product case class TopLevelDdgMetricsMetadata( userId: Option[Long], product: Product, clientApplicationId: Option[Long], countryCode: Option[String]) object TopLevelDdgMetricsMetadata { def from(request: CrMixerTweetRequest): TopLevelDdgMetricsMetadata = { TopLevelDdgMetricsMetadata( userId = request.clientContext.userId, product = request.product, clientApplicationId = request.clientContext.appId, countryCode = request.clientContext.countryCode ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/UtegTweetScribeLogger.scala ================================================ package com.twitter.cr_mixer.logging import com.twitter.cr_mixer.logging.ScribeLoggerUtils._ import com.twitter.cr_mixer.model.UtegTweetCandidateGeneratorQuery import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithScoreAndSocialProof import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.thriftscala.UtegTweetRequest import com.twitter.cr_mixer.thriftscala.UtegTweetResponse import com.twitter.cr_mixer.thriftscala.FetchCandidatesResult import com.twitter.cr_mixer.thriftscala.GetUtegTweetsScribe import com.twitter.cr_mixer.thriftscala.PerformanceMetrics import com.twitter.cr_mixer.thriftscala.UtegTweetResult import com.twitter.cr_mixer.thriftscala.UtegTweetTopLevelApiResult import com.twitter.cr_mixer.thriftscala.TweetCandidateWithMetadata import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.tracing.Trace import com.twitter.logging.Logger import com.twitter.simclusters_v2.common.UserId import com.twitter.util.Future import com.twitter.util.Stopwatch import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton case class UtegTweetScribeLogger @Inject() ( decider: CrMixerDecider, statsReceiver: StatsReceiver, @Named(ModuleNames.UtegTweetsLogger) utegTweetScribeLogger: Logger) { private val scopedStats = statsReceiver.scope("UtegTweetScribeLogger") private val topLevelApiStats = scopedStats.scope("TopLevelApi") private val upperFunnelsStats = scopedStats.scope("UpperFunnels") def scribeInitialCandidates( query: UtegTweetCandidateGeneratorQuery, getResultFn: => Future[Seq[TweetWithScoreAndSocialProof]] ): Future[Seq[TweetWithScoreAndSocialProof]] = { scribeResultsAndPerformanceMetrics( ScribeMetadata.from(query), getResultFn, convertToResultFn = convertFetchCandidatesResult ) } /** * Scribe Top Level API Request / Response and performance metrics * for the GetUtegTweetRecommendations() endpoint. */ def scribeGetUtegTweetRecommendations( request: UtegTweetRequest, startTime: Long, scribeMetadata: ScribeMetadata, getResultFn: => Future[UtegTweetResponse] ): Future[UtegTweetResponse] = { val timer = Stopwatch.start() getResultFn.onSuccess { response => if (decider.isAvailableForId( scribeMetadata.userId, DeciderConstants.upperFunnelPerStepScribeRate)) { topLevelApiStats.counter(scribeMetadata.product.originalName).incr() val latencyMs = timer().inMilliseconds val result = convertTopLevelAPIResult(request, response, startTime) val traceId = Trace.id.traceId.toLong val scribeMsg = buildScribeMessage(result, scribeMetadata, latencyMs, traceId) scribeResult(scribeMsg) } } } private def convertTopLevelAPIResult( request: UtegTweetRequest, response: UtegTweetResponse, startTime: Long ): UtegTweetResult = { UtegTweetResult.UtegTweetTopLevelApiResult( UtegTweetTopLevelApiResult( timestamp = startTime, request = request, response = response )) } private def buildScribeMessage( utegTweetResult: UtegTweetResult, scribeMetadata: ScribeMetadata, latencyMs: Long, traceId: Long ): GetUtegTweetsScribe = { GetUtegTweetsScribe( uuid = scribeMetadata.requestUUID, userId = scribeMetadata.userId, utegTweetResult = utegTweetResult, traceId = Some(traceId), performanceMetrics = Some(PerformanceMetrics(Some(latencyMs))), impressedBuckets = getImpressedBuckets(scopedStats) ) } private def scribeResult( scribeMsg: GetUtegTweetsScribe ): Unit = { publish(logger = utegTweetScribeLogger, codec = GetUtegTweetsScribe, message = scribeMsg) } private def convertFetchCandidatesResult( candidates: Seq[TweetWithScoreAndSocialProof], requestUserId: UserId ): UtegTweetResult = { val tweetCandidatesWithMetadata = candidates.map { candidate => TweetCandidateWithMetadata( tweetId = candidate.tweetId, candidateGenerationKey = None ) // do not hydrate candidateGenerationKey to save cost } UtegTweetResult.FetchCandidatesResult(FetchCandidatesResult(Some(tweetCandidatesWithMetadata))) } /** * Scribe Per-step intermediate results and performance metrics * for each step: fetch candidates, filters. */ private def scribeResultsAndPerformanceMetrics[T]( scribeMetadata: ScribeMetadata, getResultFn: => Future[T], convertToResultFn: (T, UserId) => UtegTweetResult ): Future[T] = { val timer = Stopwatch.start() getResultFn.onSuccess { input => if (decider.isAvailableForId( scribeMetadata.userId, DeciderConstants.upperFunnelPerStepScribeRate)) { upperFunnelsStats.counter(scribeMetadata.product.originalName).incr() val latencyMs = timer().inMilliseconds val result = convertToResultFn(input, scribeMetadata.userId) val traceId = Trace.id.traceId.toLong val scribeMsg = buildScribeMessage(result, scribeMetadata, latencyMs, traceId) scribeResult(scribeMsg) } } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "configapi/configapi-core", "content-recommender/thrift/src/main/thrift:thrift-scala", "cr-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", "src/scala/com/twitter/simclusters_v2/common", "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", "src/thrift/com/twitter/recos:recos-common-scala", "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/Candidate.scala ================================================ package com.twitter.cr_mixer.model import com.twitter.contentrecommender.thriftscala.TweetInfo import com.twitter.cr_mixer.thriftscala.LineItemInfo import com.twitter.simclusters_v2.common.TweetId sealed trait Candidate { val tweetId: TweetId override def hashCode: Int = tweetId.toInt } case class TweetWithCandidateGenerationInfo( tweetId: TweetId, candidateGenerationInfo: CandidateGenerationInfo) extends Candidate { def getSimilarityScore: Double = candidateGenerationInfo.similarityEngineInfo.score.getOrElse(0.0) } case class InitialCandidate( tweetId: TweetId, tweetInfo: TweetInfo, candidateGenerationInfo: CandidateGenerationInfo) extends Candidate { /** * * Get the Similarity Score of a Tweet from its CG Info. For instance, * If it is from a UnifiedTweetBasedSimilarityEngine, the score will be the weighted combined score * And if it is from a SimClustersANNSimilarityEngine, the score will be the SANN score */ def getSimilarityScore: Double = candidateGenerationInfo.similarityEngineInfo.score.getOrElse(0.0) /** * The same candidate can be generated by multiple algorithms. * During blending, candidate deduping happens. In order to retain the candidateGenerationInfo * from different algorithms, we attach them to a list of potentialReasons. */ def toBlendedCandidate( potentialReasons: Seq[CandidateGenerationInfo], ): BlendedCandidate = { BlendedCandidate( tweetId, tweetInfo, candidateGenerationInfo, potentialReasons, ) } // for experimental purposes only when bypassing interleave / ranking def toRankedCandidate(): RankedCandidate = { RankedCandidate( tweetId, tweetInfo, 0.0, // prediction score is default to 0.0 to help differentiate that it is a no-op candidateGenerationInfo, Seq(candidateGenerationInfo) ) } } case class InitialAdsCandidate( tweetId: TweetId, lineItemInfo: Seq[LineItemInfo], candidateGenerationInfo: CandidateGenerationInfo) extends Candidate { /** * * Get the Similarity Score of a Tweet from its CG Info. For instance, * If it is from a UnifiedTweetBasedSimilarityEngine, the score will be the weighted combined score * And if it is from a SimClustersANNSimilarityEngine, the score will be the SANN score */ def getSimilarityScore: Double = candidateGenerationInfo.similarityEngineInfo.score.getOrElse(0.0) /** * The same candidate can be generated by multiple algorithms. * During blending, candidate deduping happens. In order to retain the candidateGenerationInfo * from different algorithms, we attach them to a list of potentialReasons. */ def toBlendedAdsCandidate( potentialReasons: Seq[CandidateGenerationInfo], ): BlendedAdsCandidate = { BlendedAdsCandidate( tweetId, lineItemInfo, candidateGenerationInfo, potentialReasons, ) } // for experimental purposes only when bypassing interleave / ranking def toRankedAdsCandidate(): RankedAdsCandidate = { RankedAdsCandidate( tweetId, lineItemInfo, 0.0, // prediction score is default to 0.0 to help differentiate that it is a no-op candidateGenerationInfo, Seq(candidateGenerationInfo) ) } } case class BlendedCandidate( tweetId: TweetId, tweetInfo: TweetInfo, reasonChosen: CandidateGenerationInfo, potentialReasons: Seq[CandidateGenerationInfo]) extends Candidate { /** * * Get the Similarity Score of a Tweet from its CG Info. For instance, * If it is from a UnifiedTweetBasedSimilarityEngine, the score will be the weighted combined score * And if it is from a SimClustersANNSimilarityEngine, the score will be the SANN score */ def getSimilarityScore: Double = reasonChosen.similarityEngineInfo.score.getOrElse(0.0) assert(potentialReasons.contains(reasonChosen)) def toRankedCandidate(predictionScore: Double): RankedCandidate = { RankedCandidate( tweetId, tweetInfo, predictionScore, reasonChosen, potentialReasons ) } } case class BlendedAdsCandidate( tweetId: TweetId, lineItemInfo: Seq[LineItemInfo], reasonChosen: CandidateGenerationInfo, potentialReasons: Seq[CandidateGenerationInfo]) extends Candidate { /** * * Get the Similarity Score of a Tweet from its CG Info. For instance, * If it is from a UnifiedTweetBasedSimilarityEngine, the score will be the weighted combined score * And if it is from a SimClustersANNSimilarityEngine, the score will be the SANN score */ def getSimilarityScore: Double = reasonChosen.similarityEngineInfo.score.getOrElse(0.0) assert(potentialReasons.contains(reasonChosen)) def toRankedAdsCandidate(predictionScore: Double): RankedAdsCandidate = { RankedAdsCandidate( tweetId, lineItemInfo, predictionScore, reasonChosen, potentialReasons ) } } case class RankedCandidate( tweetId: TweetId, tweetInfo: TweetInfo, predictionScore: Double, reasonChosen: CandidateGenerationInfo, potentialReasons: Seq[CandidateGenerationInfo]) extends Candidate { /** * * Get the Similarity Score of a Tweet from its CG Info. For instance, * If it is from a UnifiedTweetBasedSimilarityEngine, the score will be the weighted combined score * And if it is from a SimClustersANNSimilarityEngine, the score will be the SANN score */ def getSimilarityScore: Double = reasonChosen.similarityEngineInfo.score.getOrElse(0.0) assert(potentialReasons.contains(reasonChosen)) } case class RankedAdsCandidate( tweetId: TweetId, lineItemInfo: Seq[LineItemInfo], predictionScore: Double, reasonChosen: CandidateGenerationInfo, potentialReasons: Seq[CandidateGenerationInfo]) extends Candidate { /** * * Get the Similarity Score of a Tweet from its CG Info. For instance, * If it is from a UnifiedTweetBasedSimilarityEngine, the score will be the weighted combined score * And if it is from a SimClustersANNSimilarityEngine, the score will be the SANN score */ def getSimilarityScore: Double = reasonChosen.similarityEngineInfo.score.getOrElse(0.0) assert(potentialReasons.contains(reasonChosen)) } case class TripTweetWithScore(tweetId: TweetId, score: Double) extends Candidate ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/CandidateGenerationInfo.scala ================================================ package com.twitter.cr_mixer.model import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.simclusters_v2.common.UserId import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.util.Time /*** * Tweet-level attributes. Represents the source used in candidate generation * Due to legacy reason, SourceType used to represent both SourceType and SimilarityEngineType * Moving forward, SourceType will be used for SourceType ONLY. eg., TweetFavorite, UserFollow, TwiceUserId * At the same time, We create a new SimilarityEngineType to separate them. eg., SimClustersANN * * Currently, one special case is that we have TwiceUserId as a source, which is not necessarily a "signal" * @param sourceType, e.g., SourceType.TweetFavorite, SourceType.UserFollow, SourceType.TwiceUserId * @param internalId, e.g., UserId(0L), TweetId(0L) */ case class SourceInfo( sourceType: SourceType, internalId: InternalId, sourceEventTime: Option[Time]) /*** * Tweet-level attributes. Represents the source User Graph used in candidate generation * It is an intermediate product, and will not be stored, unlike SourceInfo. * Essentially, CrMixer queries a graph, and the graph returns a list of users to be used as sources. * For instance, RealGraph, EarlyBird, FRS, Stp, etc. The underlying similarity engines such as * UTG or UTEG will leverage these sources to build candidates. * * We extended the definition of SourceType to cover both "Source Signal" and "Source Graph" * See [CrMixer] Graph Based Source Fetcher Abstraction Proposal: * * consider making both SourceInfo and GraphSourceInfo extends the same trait to * have a unified interface. */ case class GraphSourceInfo( sourceType: SourceType, seedWithScores: Map[UserId, Double]) /*** * Tweet-level attributes. Represents the similarity engine (the algorithm) used for * candidate generation along with their metadata. * @param similarityEngineType, e.g., SimClustersANN, UserTweetGraph * @param modelId. e.g., UserTweetGraphConsumerEmbedding_ALL_20210708 * @param score - a score generated by this sim engine */ case class SimilarityEngineInfo( similarityEngineType: SimilarityEngineType, modelId: Option[String], // ModelId can be a None. e.g., UTEG, UnifiedTweetBasedSE. etc score: Option[Double]) /**** * Tweet-level attributes. A combination for both SourceInfo and SimilarityEngineInfo * SimilarityEngine is a composition, and it can be composed by many leaf Similarity Engines. * For instance, the TweetBasedUnified SE could be a composition of both UserTweetGraph SE, SimClustersANN SE. * Note that a SimilarityEngine (Composite) may call other SimilarityEngines (Atomic, Contributing) * to contribute to its final candidate list. We track these Contributing SEs in the contributingSimilarityEngines list * * @param sourceInfoOpt - this is optional as many consumerBased CG does not have a source * @param similarityEngineInfo - the similarity engine used in Candidate Generation (eg., TweetBasedUnifiedSE). It can be an atomic SE or an composite SE * @param contributingSimilarityEngines - only composite SE will have it (e.g., SANNN, UTG). Otherwise it is an empty Seq. All contributing SEs mst be atomic */ case class CandidateGenerationInfo( sourceInfoOpt: Option[SourceInfo], similarityEngineInfo: SimilarityEngineInfo, contributingSimilarityEngines: Seq[SimilarityEngineInfo]) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/CandidateGeneratorQuery.scala ================================================ package com.twitter.cr_mixer.model import com.twitter.core_workflows.user_model.thriftscala.UserState import com.twitter.cr_mixer.thriftscala.Product import com.twitter.product_mixer.core.thriftscala.ClientContext import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.common.UserId import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.simclusters_v2.thriftscala.TopicId import com.twitter.timelines.configapi.Params sealed trait CandidateGeneratorQuery { val product: Product val maxNumResults: Int val impressedTweetList: Set[TweetId] val params: Params val requestUUID: Long } sealed trait HasUserId { val userId: UserId } case class CrCandidateGeneratorQuery( userId: UserId, product: Product, userState: UserState, maxNumResults: Int, impressedTweetList: Set[TweetId], params: Params, requestUUID: Long, languageCode: Option[String] = None) extends CandidateGeneratorQuery with HasUserId case class UtegTweetCandidateGeneratorQuery( userId: UserId, product: Product, userState: UserState, maxNumResults: Int, impressedTweetList: Set[TweetId], params: Params, requestUUID: Long) extends CandidateGeneratorQuery with HasUserId case class RelatedTweetCandidateGeneratorQuery( internalId: InternalId, clientContext: ClientContext, // To scribe LogIn/LogOut requests product: Product, maxNumResults: Int, impressedTweetList: Set[TweetId], params: Params, requestUUID: Long) extends CandidateGeneratorQuery case class RelatedVideoTweetCandidateGeneratorQuery( internalId: InternalId, clientContext: ClientContext, // To scribe LogIn/LogOut requests product: Product, maxNumResults: Int, impressedTweetList: Set[TweetId], params: Params, requestUUID: Long) extends CandidateGeneratorQuery case class FrsTweetCandidateGeneratorQuery( userId: UserId, product: Product, maxNumResults: Int, impressedUserList: Set[UserId], impressedTweetList: Set[TweetId], params: Params, languageCodeOpt: Option[String] = None, countryCodeOpt: Option[String] = None, requestUUID: Long) extends CandidateGeneratorQuery case class AdsCandidateGeneratorQuery( userId: UserId, product: Product, userState: UserState, maxNumResults: Int, params: Params, requestUUID: Long) case class TopicTweetCandidateGeneratorQuery( userId: UserId, topicIds: Set[TopicId], product: Product, maxNumResults: Int, impressedTweetList: Set[TweetId], params: Params, requestUUID: Long, isVideoOnly: Boolean) extends CandidateGeneratorQuery ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/EarlybirdSimilarityEngineType.scala ================================================ package com.twitter.cr_mixer.model sealed trait EarlybirdSimilarityEngineType object EarlybirdSimilarityEngineType_RecencyBased extends EarlybirdSimilarityEngineType object EarlybirdSimilarityEngineType_ModelBased extends EarlybirdSimilarityEngineType object EarlybirdSimilarityEngineType_TensorflowBased extends EarlybirdSimilarityEngineType ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/HealthThreshold.scala ================================================ package com.twitter.cr_mixer.model object HealthThreshold { object Enum extends Enumeration { val Off: Value = Value(1) val Moderate: Value = Value(2) val Strict: Value = Value(3) val Stricter: Value = Value(4) val StricterPlus: Value = Value(5) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/ModelConfig.scala ================================================ package com.twitter.cr_mixer.model /** * A Configuration class for all Model Based Candidate Sources. * * The Model Name Guideline. Please your modelId as "Algorithm_Product_Date" * If your model is used for multiple product surfaces, name it as all * Don't name your algorithm as MBCG. All the algorithms here are MBCG =.= * * Don't forgot to add your new models into allHnswANNSimilarityEngineModelIds list. */ object ModelConfig { // Offline SimClusters CG Experiment related Model Ids val OfflineInterestedInFromKnownFor2020: String = "OfflineIIKF_ALL_20220414" val OfflineInterestedInFromKnownFor2020Hl0El15: String = "OfflineIIKF_ALL_20220414_Hl0_El15" val OfflineInterestedInFromKnownFor2020Hl2El15: String = "OfflineIIKF_ALL_20220414_Hl2_El15" val OfflineInterestedInFromKnownFor2020Hl2El50: String = "OfflineIIKF_ALL_20220414_Hl2_El50" val OfflineInterestedInFromKnownFor2020Hl8El50: String = "OfflineIIKF_ALL_20220414_Hl8_El50" val OfflineMTSConsumerEmbeddingsFav90P20M: String = "OfflineMTSConsumerEmbeddingsFav90P20M_ALL_20220414" // Twhin Model Ids val ConsumerBasedTwHINRegularUpdateAll20221024: String = "ConsumerBasedTwHINRegularUpdate_All_20221024" // Averaged Twhin Model Ids val TweetBasedTwHINRegularUpdateAll20221024: String = "TweetBasedTwHINRegularUpdate_All_20221024" // Collaborative Filtering Twhin Model Ids val TwhinCollabFilterForFollow: String = "TwhinCollabFilterForFollow" val TwhinCollabFilterForEngagement: String = "TwhinCollabFilterForEngagement" val TwhinMultiClusterForFollow: String = "TwhinMultiClusterForFollow" val TwhinMultiClusterForEngagement: String = "TwhinMultiClusterForEngagement" // Two Tower model Ids val TwoTowerFavALL20220808: String = "TwoTowerFav_ALL_20220808" // Debugger Demo-Only Model Ids val DebuggerDemo: String = "DebuggerDemo" // ColdStartLookalike - this is not really a model name, it is as a placeholder to // indicate ColdStartLookalike candidate source, which is currently being pluged into // CustomizedRetrievalCandidateGeneration temporarily. val ColdStartLookalikeModelName: String = "ConsumersBasedUtgColdStartLookalike20220707" // consumersBasedUTG-RealGraphOon Model Id val ConsumersBasedUtgRealGraphOon20220705: String = "ConsumersBasedUtgRealGraphOon_All_20220705" // consumersBasedUAG-RealGraphOon Model Id val ConsumersBasedUagRealGraphOon20221205: String = "ConsumersBasedUagRealGraphOon_All_20221205" // FTR val OfflineFavDecayedSum: String = "OfflineFavDecayedSum" val OfflineFtrAt5Pop1000RnkDcy11: String = "OfflineFtrAt5Pop1000RnkDcy11" val OfflineFtrAt5Pop10000RnkDcy11: String = "OfflineFtrAt5Pop10000RnkDcy11" // All Model Ids of HnswANNSimilarityEngines val allHnswANNSimilarityEngineModelIds = Seq( ConsumerBasedTwHINRegularUpdateAll20221024, TwoTowerFavALL20220808, DebuggerDemo ) val ConsumerLogFavBasedInterestedInEmbedding: String = "ConsumerLogFavBasedInterestedIn_ALL_20221228" val ConsumerFollowBasedInterestedInEmbedding: String = "ConsumerFollowBasedInterestedIn_ALL_20221228" val RetweetBasedDiffusion: String = "RetweetBasedDiffusion" } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/ModuleNames.scala ================================================ package com.twitter.cr_mixer.model /** * Define name annotated module names here */ object ModuleNames { final val FrsStore = "FrsStore" final val UssStore = "UssStore" final val UssStratoColumn = "UssStratoColumn" final val RsxStore = "RsxStore" final val RmsTweetLogFavLongestL2EmbeddingStore = "RmsTweetLogFavLongestL2EmbeddingStore" final val RmsUserFavBasedProducerEmbeddingStore = "RmsUserFavBasedProducerEmbeddingStore" final val RmsUserLogFavInterestedInEmbeddingStore = "RmsUserLogFavInterestedInEmbeddingStore" final val RmsUserFollowInterestedInEmbeddingStore = "RmsUserFollowInterestedInEmbeddingStore" final val StpStore = "StpStore" final val TwiceClustersMembersStore = "TwiceClustersMembersStore" final val TripCandidateStore = "TripCandidateStore" final val ConsumerEmbeddingBasedTripSimilarityEngine = "ConsumerEmbeddingBasedTripSimilarityEngine" final val ConsumerEmbeddingBasedTwHINANNSimilarityEngine = "ConsumerEmbeddingBasedTwHINANNSimilarityEngine" final val ConsumerEmbeddingBasedTwoTowerANNSimilarityEngine = "ConsumerEmbeddingBasedTwoTowerANNSimilarityEngine" final val ConsumersBasedUserAdGraphSimilarityEngine = "ConsumersBasedUserAdGraphSimilarityEngine" final val ConsumersBasedUserVideoGraphSimilarityEngine = "ConsumersBasedUserVideoGraphSimilarityEngine" final val ConsumerBasedWalsSimilarityEngine = "ConsumerBasedWalsSimilarityEngine" final val TweetBasedTwHINANNSimilarityEngine = "TweetBasedTwHINANNSimilarityEngine" final val SimClustersANNSimilarityEngine = "SimClustersANNSimilarityEngine" final val ProdSimClustersANNServiceClientName = "ProdSimClustersANNServiceClient" final val ExperimentalSimClustersANNServiceClientName = "ExperimentalSimClustersANNServiceClient" final val SimClustersANNServiceClientName1 = "SimClustersANNServiceClient1" final val SimClustersANNServiceClientName2 = "SimClustersANNServiceClient2" final val SimClustersANNServiceClientName3 = "SimClustersANNServiceClient3" final val SimClustersANNServiceClientName5 = "SimClustersANNServiceClient5" final val SimClustersANNServiceClientName4 = "SimClustersANNServiceClient4" final val UnifiedCache = "unifiedCache" final val MLScoreCache = "mlScoreCache" final val TweetRecommendationResultsCache = "tweetRecommendationResultsCache" final val EarlybirdTweetsCache = "earlybirdTweetsCache" final val EarlybirdRecencyBasedWithoutRetweetsRepliesTweetsCache = "earlybirdTweetsWithoutRetweetsRepliesCacheStore" final val EarlybirdRecencyBasedWithRetweetsRepliesTweetsCache = "earlybirdTweetsWithRetweetsRepliesCacheStore" final val AbDeciderLogger = "abDeciderLogger" final val TopLevelApiDdgMetricsLogger = "topLevelApiDdgMetricsLogger" final val TweetRecsLogger = "tweetRecsLogger" final val BlueVerifiedTweetRecsLogger = "blueVerifiedTweetRecsLogger" final val RelatedTweetsLogger = "relatedTweetsLogger" final val UtegTweetsLogger = "utegTweetsLogger" final val AdsRecommendationsLogger = "adsRecommendationLogger" final val OfflineSimClustersANNInterestedInSimilarityEngine = "OfflineSimClustersANNInterestedInSimilarityEngine" final val RealGraphOonStore = "RealGraphOonStore" final val RealGraphInStore = "RealGraphInStore" final val OfflineTweet2020CandidateStore = "OfflineTweet2020CandidateStore" final val OfflineTweet2020Hl0El15CandidateStore = "OfflineTweet2020Hl0El15CandidateStore" final val OfflineTweet2020Hl2El15CandidateStore = "OfflineTweet2020Hl2El15CandidateStore" final val OfflineTweet2020Hl2El50CandidateStore = "OfflineTweet2020Hl2El50CandidateStore" final val OfflineTweet2020Hl8El50CandidateStore = "OfflineTweet2020Hl8El50CandidateStore" final val OfflineTweetMTSCandidateStore = "OfflineTweetMTSCandidateStore" final val OfflineFavDecayedSumCandidateStore = "OfflineFavDecayedSumCandidateStore" final val OfflineFtrAt5Pop1000RankDecay11CandidateStore = "OfflineFtrAt5Pop1000RankDecay11CandidateStore" final val OfflineFtrAt5Pop10000RankDecay11CandidateStore = "OfflineFtrAt5Pop10000RankDecay11CandidateStore" final val TwhinCollabFilterStratoStoreForFollow = "TwhinCollabFilterStratoStoreForFollow" final val TwhinCollabFilterStratoStoreForEngagement = "TwhinCollabFilterStratoStoreForEngagement" final val TwhinMultiClusterStratoStoreForFollow = "TwhinMultiClusterStratoStoreForFollow" final val TwhinMultiClusterStratoStoreForEngagement = "TwhinMultiClusterStratoStoreForEngagement" final val ProducerBasedUserAdGraphSimilarityEngine = "ProducerBasedUserAdGraphSimilarityEngine" final val ProducerBasedUserTweetGraphSimilarityEngine = "ProducerBasedUserTweetGraphSimilarityEngine" final val ProducerBasedUnifiedSimilarityEngine = "ProducerBasedUnifiedSimilarityEngine" final val TweetBasedUserAdGraphSimilarityEngine = "TweetBasedUserAdGraphSimilarityEngine" final val TweetBasedUserTweetGraphSimilarityEngine = "TweetBasedUserTweetGraphSimilarityEngine" final val TweetBasedUserVideoGraphSimilarityEngine = "TweetBasedUserVideoGraphSimilarityEngine" final val TweetBasedQigSimilarityEngine = "TweetBasedQigSimilarityEngine" final val TweetBasedUnifiedSimilarityEngine = "TweetBasedUnifiedSimilarityEngine" final val TwhinCollabFilterSimilarityEngine = "TwhinCollabFilterSimilarityEngine" final val ConsumerBasedUserTweetGraphStore = "ConsumerBasedUserTweetGraphStore" final val ConsumerBasedUserVideoGraphStore = "ConsumerBasedUserVideoGraphStore" final val ConsumerBasedUserAdGraphStore = "ConsumerBasedUserAdGraphStore" final val UserTweetEntityGraphSimilarityEngine = "UserTweetEntityGraphSimilarityEngine" final val CertoTopicTweetSimilarityEngine = "CertoTopicTweetSimilarityEngine" final val CertoStratoStoreName = "CertoStratoStore" final val SkitTopicTweetSimilarityEngine = "SkitTopicTweetSimilarityEngine" final val SkitHighPrecisionTopicTweetSimilarityEngine = "SkitHighPrecisionTopicTweetSimilarityEngine" final val SkitStratoStoreName = "SkitStratoStore" final val HomeNaviGRPCClient = "HomeNaviGRPCClient" final val AdsFavedNaviGRPCClient = "AdsFavedNaviGRPCClient" final val AdsMonetizableNaviGRPCClient = "AdsMonetizableNaviGRPCClient" final val RetweetBasedDiffusionRecsMhStore = "RetweetBasedDiffusionRecsMhStore" final val DiffusionBasedSimilarityEngine = "DiffusionBasedSimilarityEngine" final val BlueVerifiedAnnotationStore = "BlueVerifiedAnnotationStore" } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TopicTweetWithScore.scala ================================================ package com.twitter.cr_mixer.model import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.simclusters_v2.common.TweetId /*** * Bind a tweetId with a raw score generated from one single Similarity Engine * @param similarityEngineType, which underlying topic source the topic tweet is from */ case class TopicTweetWithScore( tweetId: TweetId, score: Double, similarityEngineType: SimilarityEngineType) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithAuthor.scala ================================================ package com.twitter.cr_mixer.model import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.common.UserId case class TweetWithAuthor(tweetId: TweetId, authorId: UserId) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithScore.scala ================================================ package com.twitter.cr_mixer.model import com.twitter.simclusters_v2.common.TweetId /*** * Bind a tweetId with a raw score generated from one single Similarity Engine */ case class TweetWithScore(tweetId: TweetId, score: Double) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithScoreAndSocialProof.scala ================================================ package com.twitter.cr_mixer.model import com.twitter.simclusters_v2.common.TweetId import com.twitter.recos.recos_common.thriftscala.SocialProofType /*** * Bind a tweetId with a raw score and social proofs by type */ case class TweetWithScoreAndSocialProof( tweetId: TweetId, score: Double, socialProofByType: Map[SocialProofType, Seq[Long]]) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ActivePromotedTweetStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.bijection.thrift.CompactThriftCodec import com.twitter.ads.entities.db.thriftscala.LineItemObjective import com.twitter.bijection.Injection import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.thriftscala.LineItemInfo import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.finagle.stats.StatsReceiver import com.twitter.hermit.store.common.ObservedCachedReadableStore import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.inject.TwitterModule import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataType import com.twitter.ml.api.Feature import com.twitter.ml.api.GeneralTensor import com.twitter.ml.api.RichDataRecord import com.twitter.relevance_platform.common.injection.LZ4Injection import com.twitter.relevance_platform.common.injection.SeqObjectInjection import com.twitter.simclusters_v2.common.TweetId import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams import com.twitter.storehaus.ReadableStore import com.twitter.storehaus_internal.manhattan.ManhattanRO import com.twitter.storehaus_internal.manhattan.ManhattanROConfig import com.twitter.storehaus_internal.manhattan.Revenue import com.twitter.storehaus_internal.util.ApplicationID import com.twitter.storehaus_internal.util.DatasetName import com.twitter.storehaus_internal.util.HDFSPath import com.twitter.util.Future import javax.inject.Named import scala.collection.JavaConverters._ object ActivePromotedTweetStoreModule extends TwitterModule { case class ActivePromotedTweetStore( activePromotedTweetMHStore: ReadableStore[String, DataRecord], statsReceiver: StatsReceiver) extends ReadableStore[TweetId, Seq[LineItemInfo]] { override def get(tweetId: TweetId): Future[Option[Seq[LineItemInfo]]] = { activePromotedTweetMHStore.get(tweetId.toString).map { _.map { dataRecord => val richDataRecord = new RichDataRecord(dataRecord) val lineItemIdsFeature: Feature[GeneralTensor] = new Feature.Tensor("active_promoted_tweets.line_item_ids", DataType.INT64) val lineItemObjectivesFeature: Feature[GeneralTensor] = new Feature.Tensor("active_promoted_tweets.line_item_objectives", DataType.INT64) val lineItemIdsTensor: GeneralTensor = richDataRecord.getFeatureValue(lineItemIdsFeature) val lineItemObjectivesTensor: GeneralTensor = richDataRecord.getFeatureValue(lineItemObjectivesFeature) val lineItemIds: Seq[Long] = if (lineItemIdsTensor.getSetField == GeneralTensor._Fields.INT64_TENSOR && lineItemIdsTensor.getInt64Tensor.isSetLongs) { lineItemIdsTensor.getInt64Tensor.getLongs.asScala.map(_.toLong) } else Seq.empty val lineItemObjectives: Seq[LineItemObjective] = if (lineItemObjectivesTensor.getSetField == GeneralTensor._Fields.INT64_TENSOR && lineItemObjectivesTensor.getInt64Tensor.isSetLongs) { lineItemObjectivesTensor.getInt64Tensor.getLongs.asScala.map(objective => LineItemObjective(objective.toInt)) } else Seq.empty val lineItemInfo = if (lineItemIds.size == lineItemObjectives.size) { lineItemIds.zipWithIndex.map { case (lineItemId, index) => LineItemInfo( lineItemId = lineItemId, lineItemObjective = lineItemObjectives(index) ) } } else Seq.empty lineItemInfo } } } } @Provides @Singleton def providesActivePromotedTweetStore( manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, crMixerStatsReceiver: StatsReceiver ): ReadableStore[TweetId, Seq[LineItemInfo]] = { val mhConfig = new ManhattanROConfig { val hdfsPath = HDFSPath("") val applicationID = ApplicationID("ads_bigquery_features") val datasetName = DatasetName("active_promoted_tweets") val cluster = Revenue override def statsReceiver: StatsReceiver = crMixerStatsReceiver.scope("active_promoted_tweets_mh") } val mhStore: ReadableStore[String, DataRecord] = ManhattanRO .getReadableStoreWithMtls[String, DataRecord]( mhConfig, manhattanKVClientMtlsParams )( implicitly[Injection[String, Array[Byte]]], CompactThriftCodec[DataRecord] ) val underlyingStore = ActivePromotedTweetStore(mhStore, crMixerStatsReceiver.scope("ActivePromotedTweetStore")) val memcachedStore = ObservedMemcachedReadableStore.fromCacheClient( backingStore = underlyingStore, cacheClient = crMixerUnifiedCacheClient, ttl = 60.minutes, asyncUpdate = false )( valueInjection = LZ4Injection.compose(SeqObjectInjection[LineItemInfo]()), statsReceiver = crMixerStatsReceiver.scope("memCachedActivePromotedTweetStore"), keyToString = { k: TweetId => s"apt/$k" } ) ObservedCachedReadableStore.from( memcachedStore, ttl = 30.minutes, maxKeys = 250000, // size of promoted tweet is around 200,000 windowSize = 10000L, cacheName = "active_promoted_tweet_cache", maxMultiGetSize = 20 )(crMixerStatsReceiver.scope("inMemoryCachedActivePromotedTweetStore")) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/BUILD.bazel ================================================ scala_library( sources = [ "*.scala", "core/*.scala", "grpc_client/*.scala", "similarity_engine/*.scala", "source_signal/*.scala", "thrift_client/*.scala", ], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/bijection:core", "3rdparty/jvm/com/twitter/bijection:scrooge", "3rdparty/jvm/com/twitter/storehaus:core", "3rdparty/jvm/com/twitter/storehaus:memcache", "3rdparty/jvm/io/grpc:grpc-api", "3rdparty/jvm/io/grpc:grpc-auth", "3rdparty/jvm/io/grpc:grpc-core", "3rdparty/jvm/io/grpc:grpc-netty", "3rdparty/jvm/io/grpc:grpc-protobuf", "3rdparty/jvm/io/grpc:grpc-stub", "3rdparty/jvm/javax/inject:javax.inject", "3rdparty/jvm/org/scalanlp:breeze", "3rdparty/src/jvm/com/twitter/storehaus:core", "abdecider/src/main/scala", "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", "configapi/configapi-abdecider", "configapi/configapi-core", "configapi/configapi-featureswitches:v2", "content-recommender/server/src/main/scala/com/twitter/contentrecommender:cr-mixer-deps", "content-recommender/thrift/src/main/thrift:thrift-scala", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/config", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/featureswitch", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util", "cr-mixer/thrift/src/main/thrift:thrift-scala", "decider/src/main/scala", "discovery-common/src/main/scala/com/twitter/discovery/common/configapi", "featureswitches/featureswitches-core", "featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder", "finagle-internal/finagle-grpc/src/main/scala", "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", "finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers", "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-core/src/main/scala", "finatra/inject/inject-modules/src/main/scala", "finatra/inject/inject-thrift-client", "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "frigate/frigate-common:util", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/base", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/candidate", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/health", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/interests", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato", "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", "hydra/partition/thrift/src/main/thrift:thrift-scala", "hydra/root/thrift/src/main/thrift:thrift-scala", "mediaservices/commons/src/main/scala:futuretracker", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", "qig-ranker/thrift/src/main/thrift:thrift-scala", "relevance-platform/src/main/scala/com/twitter/relevance_platform/common/health_store", "relevance-platform/src/main/scala/com/twitter/relevance_platform/common/injection", "relevance-platform/thrift/src/main/thrift:thrift-scala", "representation-manager/client/src/main/scala/com/twitter/representation_manager", "representation-manager/client/src/main/scala/com/twitter/representation_manager/config", "representation-manager/server/src/main/scala/com/twitter/representation_manager/migration", "representation-manager/server/src/main/thrift:thrift-scala", "representation-scorer/server/src/main/thrift:thrift-scala", "servo/decider", "servo/util/src/main/scala", "simclusters-ann/thrift/src/main/thrift:thrift-scala", "snowflake/src/main/scala/com/twitter/snowflake/id", "src/java/com/twitter/ml/api:api-base", "src/java/com/twitter/search/queryparser/query:core-query-nodes", "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", "src/scala/com/twitter/algebird_internal/injection", "src/scala/com/twitter/cortex/ml/embeddings/common:Helpers", "src/scala/com/twitter/ml/api/embedding", "src/scala/com/twitter/ml/featurestore/lib", "src/scala/com/twitter/scalding_internal/multiformat/format", "src/scala/com/twitter/simclusters_v2/candidate_source", "src/scala/com/twitter/simclusters_v2/common", "src/scala/com/twitter/storehaus_internal/manhattan", "src/scala/com/twitter/storehaus_internal/manhattan/config", "src/scala/com/twitter/storehaus_internal/memcache", "src/scala/com/twitter/storehaus_internal/memcache/config", "src/scala/com/twitter/storehaus_internal/offline", "src/scala/com/twitter/storehaus_internal/util", "src/scala/com/twitter/topic_recos/stores", "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", "src/thrift/com/twitter/frigate:frigate-common-thrift-scala", "src/thrift/com/twitter/frigate:frigate-thrift-scala", "src/thrift/com/twitter/frigate/data_pipeline/scalding:blue_verified_annotations-scala", "src/thrift/com/twitter/hermit/stp:hermit-stp-scala", "src/thrift/com/twitter/ml/api:data-java", "src/thrift/com/twitter/ml/api:embedding-scala", "src/thrift/com/twitter/ml/featurestore:ml-feature-store-embedding-scala", "src/thrift/com/twitter/onboarding/relevance/coldstart_lookalike:coldstartlookalike-thrift-scala", "src/thrift/com/twitter/recos:recos-common-scala", "src/thrift/com/twitter/recos/user_ad_graph:user_ad_graph-scala", "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", "src/thrift/com/twitter/recos/user_tweet_graph:user_tweet_graph-scala", "src/thrift/com/twitter/recos/user_tweet_graph_plus:user_tweet_graph_plus-scala", "src/thrift/com/twitter/recos/user_video_graph:user_video_graph-scala", "src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/search/query_interaction_graph/service:qig-service-scala", "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", "src/thrift/com/twitter/topic_recos:topic_recos-thrift-scala", "src/thrift/com/twitter/trends/trip_v1:trip-tweets-thrift-scala", "src/thrift/com/twitter/tweetypie:service-scala", "src/thrift/com/twitter/twistly:twistly-scala", "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", "stitch/stitch-storehaus", "stitch/stitch-tweetypie/src/main/scala", "strato/src/main/scala/com/twitter/strato/client", "user-signal-service/thrift/src/main/thrift:thrift-scala", "util-internal/scribe/src/main/scala/com/twitter/logging", "util/util-hashing", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/BlueVerifiedAnnotationStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.google.inject.name.Named import com.twitter.inject.TwitterModule import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.model.ModuleNames import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.data_pipeline.scalding.thriftscala.BlueVerifiedAnnotationsV2 import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams import com.twitter.storehaus.ReadableStore import com.twitter.storehaus_internal.manhattan.Athena import com.twitter.storehaus_internal.manhattan.ManhattanRO import com.twitter.storehaus_internal.manhattan.ManhattanROConfig import com.twitter.storehaus_internal.util.ApplicationID import com.twitter.storehaus_internal.util.DatasetName import com.twitter.storehaus_internal.util.HDFSPath import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.hermit.store.common.ObservedCachedReadableStore object BlueVerifiedAnnotationStoreModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.BlueVerifiedAnnotationStore) def providesBlueVerifiedAnnotationStore( statsReceiver: StatsReceiver, manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, ): ReadableStore[String, BlueVerifiedAnnotationsV2] = { implicit val valueCodec = new BinaryScalaCodec(BlueVerifiedAnnotationsV2) val underlyingStore = ManhattanRO .getReadableStoreWithMtls[String, BlueVerifiedAnnotationsV2]( ManhattanROConfig( HDFSPath(""), ApplicationID("content_recommender_athena"), DatasetName("blue_verified_annotations"), Athena), manhattanKVClientMtlsParams ) ObservedCachedReadableStore.from( underlyingStore, ttl = 24.hours, maxKeys = 100000, windowSize = 10000L, cacheName = "blue_verified_annotation_cache" )(statsReceiver.scope("inMemoryCachedBlueVerifiedAnnotationStore")) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/CertoStratoStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.google.inject.name.Named import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.keyHasher import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.finagle.stats.StatsReceiver import com.twitter.hermit.store.common.ObservedCachedReadableStore import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.hermit.store.common.ObservedReadableStore import com.twitter.inject.TwitterModule import com.twitter.relevance_platform.common.injection.LZ4Injection import com.twitter.relevance_platform.common.injection.SeqObjectInjection import com.twitter.simclusters_v2.thriftscala.TopicId import com.twitter.storehaus.ReadableStore import com.twitter.strato.client.Client import com.twitter.topic_recos.stores.CertoTopicTopKTweetsStore import com.twitter.topic_recos.thriftscala.TweetWithScores object CertoStratoStoreModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.CertoStratoStoreName) def providesCertoStratoStore( @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, stratoClient: Client, statsReceiver: StatsReceiver ): ReadableStore[TopicId, Seq[TweetWithScores]] = { val certoStore = ObservedReadableStore(CertoTopicTopKTweetsStore.prodStore(stratoClient))( statsReceiver.scope(ModuleNames.CertoStratoStoreName)).mapValues { topKTweetsWithScores => topKTweetsWithScores.topTweetsByFollowerL2NormalizedCosineSimilarityScore } val memCachedStore = ObservedMemcachedReadableStore .fromCacheClient( backingStore = certoStore, cacheClient = crMixerUnifiedCacheClient, ttl = 10.minutes )( valueInjection = LZ4Injection.compose(SeqObjectInjection[TweetWithScores]()), statsReceiver = statsReceiver.scope("memcached_certo_store"), keyToString = { k => s"certo:${keyHasher.hashKey(k.toString.getBytes)}" } ) ObservedCachedReadableStore.from[TopicId, Seq[TweetWithScores]]( memCachedStore, ttl = 5.minutes, maxKeys = 100000, // ~150MB max cacheName = "certo_in_memory_cache", windowSize = 10000L )(statsReceiver.scope("certo_in_memory_cache")) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserAdGraphStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.twitter.cr_mixer.model.ModuleNames import com.twitter.inject.TwitterModule import com.twitter.recos.user_ad_graph.thriftscala.ConsumersBasedRelatedAdRequest import com.twitter.recos.user_ad_graph.thriftscala.RelatedAdResponse import com.twitter.recos.user_ad_graph.thriftscala.UserAdGraph import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import javax.inject.Named import javax.inject.Singleton object ConsumersBasedUserAdGraphStoreModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.ConsumerBasedUserAdGraphStore) def providesConsumerBasedUserAdGraphStore( userAdGraphService: UserAdGraph.MethodPerEndpoint ): ReadableStore[ConsumersBasedRelatedAdRequest, RelatedAdResponse] = { new ReadableStore[ConsumersBasedRelatedAdRequest, RelatedAdResponse] { override def get( k: ConsumersBasedRelatedAdRequest ): Future[Option[RelatedAdResponse]] = { userAdGraphService.consumersBasedRelatedAds(k).map(Some(_)) } } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserTweetGraphStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.twitter.cr_mixer.model.ModuleNames import com.twitter.inject.TwitterModule import com.twitter.recos.user_tweet_graph.thriftscala.ConsumersBasedRelatedTweetRequest import com.twitter.recos.user_tweet_graph.thriftscala.RelatedTweetResponse import com.twitter.recos.user_tweet_graph.thriftscala.UserTweetGraph import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import javax.inject.Named import javax.inject.Singleton object ConsumersBasedUserTweetGraphStoreModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.ConsumerBasedUserTweetGraphStore) def providesConsumerBasedUserTweetGraphStore( userTweetGraphService: UserTweetGraph.MethodPerEndpoint ): ReadableStore[ConsumersBasedRelatedTweetRequest, RelatedTweetResponse] = { new ReadableStore[ConsumersBasedRelatedTweetRequest, RelatedTweetResponse] { override def get( k: ConsumersBasedRelatedTweetRequest ): Future[Option[RelatedTweetResponse]] = { userTweetGraphService.consumersBasedRelatedTweets(k).map(Some(_)) } } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserVideoGraphStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.twitter.cr_mixer.model.ModuleNames import com.twitter.inject.TwitterModule import com.twitter.recos.user_video_graph.thriftscala.ConsumersBasedRelatedTweetRequest import com.twitter.recos.user_video_graph.thriftscala.RelatedTweetResponse import com.twitter.recos.user_video_graph.thriftscala.UserVideoGraph import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import javax.inject.Named import javax.inject.Singleton object ConsumersBasedUserVideoGraphStoreModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.ConsumerBasedUserVideoGraphStore) def providesConsumerBasedUserVideoGraphStore( userVideoGraphService: UserVideoGraph.MethodPerEndpoint ): ReadableStore[ConsumersBasedRelatedTweetRequest, RelatedTweetResponse] = { new ReadableStore[ConsumersBasedRelatedTweetRequest, RelatedTweetResponse] { override def get( k: ConsumersBasedRelatedTweetRequest ): Future[Option[RelatedTweetResponse]] = { userVideoGraphService.consumersBasedRelatedTweets(k).map(Some(_)) } } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/CrMixerParamConfigModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.twitter.timelines.configapi.Config import com.twitter.cr_mixer.param.CrMixerParamConfig import com.twitter.inject.TwitterModule import javax.inject.Singleton object CrMixerParamConfigModule extends TwitterModule { @Provides @Singleton def provideConfig(): Config = { CrMixerParamConfig.config } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/DiffusionStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.twitter.bijection.Injection import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.cr_mixer.model.ModuleNames import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.thriftscala.TweetsWithScore import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams import com.twitter.storehaus.ReadableStore import com.twitter.storehaus_internal.manhattan.Apollo import com.twitter.storehaus_internal.manhattan.ManhattanRO import com.twitter.storehaus_internal.manhattan.ManhattanROConfig import com.twitter.storehaus_internal.util.ApplicationID import com.twitter.storehaus_internal.util.DatasetName import com.twitter.storehaus_internal.util.HDFSPath import javax.inject.Named import javax.inject.Singleton object DiffusionStoreModule extends TwitterModule { type UserId = Long implicit val longCodec = implicitly[Injection[Long, Array[Byte]]] implicit val tweetRecsInjection: Injection[TweetsWithScore, Array[Byte]] = BinaryScalaCodec(TweetsWithScore) @Provides @Singleton @Named(ModuleNames.RetweetBasedDiffusionRecsMhStore) def retweetBasedDiffusionRecsMhStore( serviceIdentifier: ServiceIdentifier ): ReadableStore[Long, TweetsWithScore] = { val manhattanROConfig = ManhattanROConfig( HDFSPath(""), // not needed ApplicationID("cr_mixer_apollo"), DatasetName("diffusion_retweet_tweet_recs"), Apollo ) buildTweetRecsStore(serviceIdentifier, manhattanROConfig) } private def buildTweetRecsStore( serviceIdentifier: ServiceIdentifier, manhattanROConfig: ManhattanROConfig ): ReadableStore[Long, TweetsWithScore] = { ManhattanRO .getReadableStoreWithMtls[Long, TweetsWithScore]( manhattanROConfig, ManhattanKVClientMtlsParams(serviceIdentifier) )(longCodec, tweetRecsInjection) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/EarlybirdRecencyBasedCandidateStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.util.EarlybirdSearchUtil.EarlybirdClientId import com.twitter.cr_mixer.util.EarlybirdSearchUtil.FacetsToFetch import com.twitter.cr_mixer.util.EarlybirdSearchUtil.GetCollectorTerminationParams import com.twitter.cr_mixer.util.EarlybirdSearchUtil.GetEarlybirdQuery import com.twitter.cr_mixer.util.EarlybirdSearchUtil.MetadataOptions import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.SeqLongInjection import com.twitter.hashing.KeyHasher import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.inject.TwitterModule import com.twitter.search.common.query.thriftjava.thriftscala.CollectorParams import com.twitter.search.earlybird.thriftscala.EarlybirdRequest import com.twitter.search.earlybird.thriftscala.EarlybirdResponseCode import com.twitter.search.earlybird.thriftscala.EarlybirdService import com.twitter.search.earlybird.thriftscala.ThriftSearchQuery import com.twitter.search.earlybird.thriftscala.ThriftSearchRankingMode import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.common.UserId import com.twitter.storehaus.ReadableStore import com.twitter.util.Duration import com.twitter.util.Future import javax.inject.Named object EarlybirdRecencyBasedCandidateStoreModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.EarlybirdRecencyBasedWithoutRetweetsRepliesTweetsCache) def providesEarlybirdRecencyBasedWithoutRetweetsRepliesCandidateStore( statsReceiver: StatsReceiver, earlybirdSearchClient: EarlybirdService.MethodPerEndpoint, @Named(ModuleNames.EarlybirdTweetsCache) earlybirdRecencyBasedTweetsCache: MemcachedClient, timeoutConfig: TimeoutConfig ): ReadableStore[UserId, Seq[TweetId]] = { val stats = statsReceiver.scope("EarlybirdRecencyBasedWithoutRetweetsRepliesCandidateStore") val underlyingStore = new ReadableStore[UserId, Seq[TweetId]] { override def get(userId: UserId): Future[Option[Seq[TweetId]]] = { // Home based EB filters out retweets and replies val earlybirdRequest = buildEarlybirdRequest( userId, FilterOutRetweetsAndReplies, DefaultMaxNumTweetPerUser, timeoutConfig.earlybirdServerTimeout) getEarlybirdSearchResult(earlybirdSearchClient, earlybirdRequest, stats) } } ObservedMemcachedReadableStore.fromCacheClient( backingStore = underlyingStore, cacheClient = earlybirdRecencyBasedTweetsCache, ttl = MemcacheKeyTimeToLiveDuration, asyncUpdate = true )( valueInjection = SeqLongInjection, statsReceiver = statsReceiver.scope("earlybird_recency_based_tweets_home_memcache"), keyToString = { k => f"uEBRBHM:${keyHasher.hashKey(k.toString.getBytes)}%X" // prefix = EarlyBirdRecencyBasedHoMe } ) } @Provides @Singleton @Named(ModuleNames.EarlybirdRecencyBasedWithRetweetsRepliesTweetsCache) def providesEarlybirdRecencyBasedWithRetweetsRepliesCandidateStore( statsReceiver: StatsReceiver, earlybirdSearchClient: EarlybirdService.MethodPerEndpoint, @Named(ModuleNames.EarlybirdTweetsCache) earlybirdRecencyBasedTweetsCache: MemcachedClient, timeoutConfig: TimeoutConfig ): ReadableStore[UserId, Seq[TweetId]] = { val stats = statsReceiver.scope("EarlybirdRecencyBasedWithRetweetsRepliesCandidateStore") val underlyingStore = new ReadableStore[UserId, Seq[TweetId]] { override def get(userId: UserId): Future[Option[Seq[TweetId]]] = { val earlybirdRequest = buildEarlybirdRequest( userId, // Notifications based EB keeps retweets and replies NotFilterOutRetweetsAndReplies, DefaultMaxNumTweetPerUser, processingTimeout = timeoutConfig.earlybirdServerTimeout ) getEarlybirdSearchResult(earlybirdSearchClient, earlybirdRequest, stats) } } ObservedMemcachedReadableStore.fromCacheClient( backingStore = underlyingStore, cacheClient = earlybirdRecencyBasedTweetsCache, ttl = MemcacheKeyTimeToLiveDuration, asyncUpdate = true )( valueInjection = SeqLongInjection, statsReceiver = statsReceiver.scope("earlybird_recency_based_tweets_notifications_memcache"), keyToString = { k => f"uEBRBN:${keyHasher.hashKey(k.toString.getBytes)}%X" // prefix = EarlyBirdRecencyBasedNotifications } ) } private val keyHasher: KeyHasher = KeyHasher.FNV1A_64 /** * Note the DefaultMaxNumTweetPerUser is used to adjust the result size per cache entry. * If the value changes, it will increase the size of the memcache. */ private val DefaultMaxNumTweetPerUser: Int = 100 private val FilterOutRetweetsAndReplies = true private val NotFilterOutRetweetsAndReplies = false private val MemcacheKeyTimeToLiveDuration: Duration = Duration.fromMinutes(15) private def buildEarlybirdRequest( seedUserId: UserId, filterOutRetweetsAndReplies: Boolean, maxNumTweetsPerSeedUser: Int, processingTimeout: Duration ): EarlybirdRequest = EarlybirdRequest( searchQuery = getThriftSearchQuery( seedUserId = seedUserId, filterOutRetweetsAndReplies = filterOutRetweetsAndReplies, maxNumTweetsPerSeedUser = maxNumTweetsPerSeedUser, processingTimeout = processingTimeout ), clientId = Some(EarlybirdClientId), timeoutMs = processingTimeout.inMilliseconds.intValue(), getOlderResults = Some(false), adjustedProtectedRequestParams = None, adjustedFullArchiveRequestParams = None, getProtectedTweetsOnly = Some(false), skipVeryRecentTweets = true, ) private def getThriftSearchQuery( seedUserId: UserId, filterOutRetweetsAndReplies: Boolean, maxNumTweetsPerSeedUser: Int, processingTimeout: Duration ): ThriftSearchQuery = ThriftSearchQuery( serializedQuery = GetEarlybirdQuery( None, None, Set.empty, filterOutRetweetsAndReplies ).map(_.serialize), fromUserIDFilter64 = Some(Seq(seedUserId)), numResults = maxNumTweetsPerSeedUser, rankingMode = ThriftSearchRankingMode.Recency, collectorParams = Some( CollectorParams( // numResultsToReturn defines how many results each EB shard will return to search root numResultsToReturn = maxNumTweetsPerSeedUser, // terminationParams.maxHitsToProcess is used for early terminating per shard results fetching. terminationParams = GetCollectorTerminationParams(maxNumTweetsPerSeedUser, processingTimeout) )), facetFieldNames = Some(FacetsToFetch), resultMetadataOptions = Some(MetadataOptions), searchStatusIds = None ) private def getEarlybirdSearchResult( earlybirdSearchClient: EarlybirdService.MethodPerEndpoint, request: EarlybirdRequest, statsReceiver: StatsReceiver ): Future[Option[Seq[TweetId]]] = earlybirdSearchClient .search(request) .map { response => response.responseCode match { case EarlybirdResponseCode.Success => val earlybirdSearchResult = response.searchResults .map { _.results .map(searchResult => searchResult.id) } statsReceiver.scope("result").stat("size").add(earlybirdSearchResult.size) earlybirdSearchResult case e => statsReceiver.scope("failures").counter(e.getClass.getSimpleName).incr() Some(Seq.empty) } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/EmbeddingStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.twitter.bijection.Injection import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.bijection.scrooge.CompactScalaCodec import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.inject.TwitterModule import com.twitter.ml.api.{thriftscala => api} import com.twitter.simclusters_v2.thriftscala.CandidateTweetsList import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams import com.twitter.storehaus.ReadableStore import com.twitter.storehaus_internal.manhattan.Apollo import com.twitter.storehaus_internal.manhattan.ManhattanRO import com.twitter.storehaus_internal.manhattan.ManhattanROConfig import com.twitter.storehaus_internal.util.ApplicationID import com.twitter.storehaus_internal.util.DatasetName import com.twitter.storehaus_internal.util.HDFSPath import javax.inject.Named import javax.inject.Singleton object EmbeddingStoreModule extends TwitterModule { type UserId = Long implicit val mbcgUserEmbeddingInjection: Injection[api.Embedding, Array[Byte]] = CompactScalaCodec(api.Embedding) implicit val tweetCandidatesInjection: Injection[CandidateTweetsList, Array[Byte]] = CompactScalaCodec(CandidateTweetsList) final val TwHINEmbeddingRegularUpdateMhStoreName = "TwHINEmbeddingRegularUpdateMhStore" @Provides @Singleton @Named(TwHINEmbeddingRegularUpdateMhStoreName) def twHINEmbeddingRegularUpdateMhStore( serviceIdentifier: ServiceIdentifier ): ReadableStore[InternalId, api.Embedding] = { val binaryEmbeddingInjection: Injection[api.Embedding, Array[Byte]] = BinaryScalaCodec(api.Embedding) val longCodec = implicitly[Injection[Long, Array[Byte]]] ManhattanRO .getReadableStoreWithMtls[TweetId, api.Embedding]( ManhattanROConfig( HDFSPath(""), // not needed ApplicationID("cr_mixer_apollo"), DatasetName("twhin_regular_update_tweet_embedding_apollo"), Apollo ), ManhattanKVClientMtlsParams(serviceIdentifier) )(longCodec, binaryEmbeddingInjection).composeKeyMapping[InternalId] { case InternalId.TweetId(tweetId) => tweetId case _ => throw new UnsupportedOperationException("Invalid Internal Id") } } final val ConsumerBasedTwHINEmbeddingRegularUpdateMhStoreName = "ConsumerBasedTwHINEmbeddingRegularUpdateMhStore" @Provides @Singleton @Named(ConsumerBasedTwHINEmbeddingRegularUpdateMhStoreName) def consumerBasedTwHINEmbeddingRegularUpdateMhStore( serviceIdentifier: ServiceIdentifier ): ReadableStore[InternalId, api.Embedding] = { val binaryEmbeddingInjection: Injection[api.Embedding, Array[Byte]] = BinaryScalaCodec(api.Embedding) val longCodec = implicitly[Injection[Long, Array[Byte]]] ManhattanRO .getReadableStoreWithMtls[UserId, api.Embedding]( ManhattanROConfig( HDFSPath(""), // not needed ApplicationID("cr_mixer_apollo"), DatasetName("twhin_user_embedding_regular_update_apollo"), Apollo ), ManhattanKVClientMtlsParams(serviceIdentifier) )(longCodec, binaryEmbeddingInjection).composeKeyMapping[InternalId] { case InternalId.UserId(userId) => userId case _ => throw new UnsupportedOperationException("Invalid Internal Id") } } final val TwoTowerFavConsumerEmbeddingMhStoreName = "TwoTowerFavConsumerEmbeddingMhStore" @Provides @Singleton @Named(TwoTowerFavConsumerEmbeddingMhStoreName) def twoTowerFavConsumerEmbeddingMhStore( serviceIdentifier: ServiceIdentifier ): ReadableStore[InternalId, api.Embedding] = { val binaryEmbeddingInjection: Injection[api.Embedding, Array[Byte]] = BinaryScalaCodec(api.Embedding) val longCodec = implicitly[Injection[Long, Array[Byte]]] ManhattanRO .getReadableStoreWithMtls[UserId, api.Embedding]( ManhattanROConfig( HDFSPath(""), // not needed ApplicationID("cr_mixer_apollo"), DatasetName("two_tower_fav_user_embedding_apollo"), Apollo ), ManhattanKVClientMtlsParams(serviceIdentifier) )(longCodec, binaryEmbeddingInjection).composeKeyMapping[InternalId] { case InternalId.UserId(userId) => userId case _ => throw new UnsupportedOperationException("Invalid Internal Id") } } final val DebuggerDemoUserEmbeddingMhStoreName = "DebuggerDemoUserEmbeddingMhStoreName" @Provides @Singleton @Named(DebuggerDemoUserEmbeddingMhStoreName) def debuggerDemoUserEmbeddingStore( serviceIdentifier: ServiceIdentifier ): ReadableStore[InternalId, api.Embedding] = { // This dataset is from src/scala/com/twitter/wtf/beam/bq_embedding_export/sql/MlfExperimentalUserEmbeddingScalaDataset.sql // Change the above sql if you want to use a diff embedding val manhattanROConfig = ManhattanROConfig( HDFSPath(""), // not needed ApplicationID("cr_mixer_apollo"), DatasetName("experimental_user_embedding"), Apollo ) buildUserEmbeddingStore(serviceIdentifier, manhattanROConfig) } final val DebuggerDemoTweetEmbeddingMhStoreName = "DebuggerDemoTweetEmbeddingMhStore" @Provides @Singleton @Named(DebuggerDemoTweetEmbeddingMhStoreName) def debuggerDemoTweetEmbeddingStore( serviceIdentifier: ServiceIdentifier ): ReadableStore[InternalId, api.Embedding] = { // This dataset is from src/scala/com/twitter/wtf/beam/bq_embedding_export/sql/MlfExperimentalTweetEmbeddingScalaDataset.sql // Change the above sql if you want to use a diff embedding val manhattanROConfig = ManhattanROConfig( HDFSPath(""), // not needed ApplicationID("cr_mixer_apollo"), DatasetName("experimental_tweet_embedding"), Apollo ) buildTweetEmbeddingStore(serviceIdentifier, manhattanROConfig) } private def buildUserEmbeddingStore( serviceIdentifier: ServiceIdentifier, manhattanROConfig: ManhattanROConfig ): ReadableStore[InternalId, api.Embedding] = { val binaryEmbeddingInjection: Injection[api.Embedding, Array[Byte]] = BinaryScalaCodec(api.Embedding) val longCodec = implicitly[Injection[Long, Array[Byte]]] ManhattanRO .getReadableStoreWithMtls[UserId, api.Embedding]( manhattanROConfig, ManhattanKVClientMtlsParams(serviceIdentifier) )(longCodec, binaryEmbeddingInjection).composeKeyMapping[InternalId] { case InternalId.UserId(userId) => userId case _ => throw new UnsupportedOperationException("Invalid Internal Id") } } private def buildTweetEmbeddingStore( serviceIdentifier: ServiceIdentifier, manhattanROConfig: ManhattanROConfig ): ReadableStore[InternalId, api.Embedding] = { val binaryEmbeddingInjection: Injection[api.Embedding, Array[Byte]] = BinaryScalaCodec(api.Embedding) val longCodec = implicitly[Injection[Long, Array[Byte]]] ManhattanRO .getReadableStoreWithMtls[TweetId, api.Embedding]( manhattanROConfig, ManhattanKVClientMtlsParams(serviceIdentifier) )(longCodec, binaryEmbeddingInjection).composeKeyMapping[InternalId] { case InternalId.TweetId(tweetId) => tweetId case _ => throw new UnsupportedOperationException("Invalid Internal Id") } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/FrsStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.source_signal.FrsStore import com.twitter.cr_mixer.source_signal.FrsStore.FrsQueryResult import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService import com.twitter.hermit.store.common.ObservedReadableStore import com.twitter.storehaus.ReadableStore import javax.inject.Named object FrsStoreModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.FrsStore) def providesFrsStore( frsClient: FollowRecommendationsThriftService.MethodPerEndpoint, statsReceiver: StatsReceiver, decider: CrMixerDecider ): ReadableStore[FrsStore.Query, Seq[FrsQueryResult]] = { ObservedReadableStore(FrsStore(frsClient, statsReceiver, decider))( statsReceiver.scope("follow_recommendations_store")) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/MHMtlsParamsModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.inject.TwitterModule import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams import javax.inject.Singleton object MHMtlsParamsModule extends TwitterModule { @Singleton @Provides def providesManhattanMtlsParams( serviceIdentifier: ServiceIdentifier ): ManhattanKVClientMtlsParams = { ManhattanKVClientMtlsParams(serviceIdentifier) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/OfflineCandidateStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.twitter.bijection.Injection import com.twitter.bijection.scrooge.CompactScalaCodec import com.twitter.cr_mixer.model.ModuleNames import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.thriftscala.CandidateTweetsList import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams import com.twitter.storehaus.ReadableStore import com.twitter.storehaus_internal.manhattan.Apollo import com.twitter.storehaus_internal.manhattan.ManhattanRO import com.twitter.storehaus_internal.manhattan.ManhattanROConfig import com.twitter.storehaus_internal.util.ApplicationID import com.twitter.storehaus_internal.util.DatasetName import com.twitter.storehaus_internal.util.HDFSPath import javax.inject.Named import javax.inject.Singleton object OfflineCandidateStoreModule extends TwitterModule { type UserId = Long implicit val tweetCandidatesInjection: Injection[CandidateTweetsList, Array[Byte]] = CompactScalaCodec(CandidateTweetsList) @Provides @Singleton @Named(ModuleNames.OfflineTweet2020CandidateStore) def offlineTweet2020CandidateMhStore( serviceIdentifier: ServiceIdentifier ): ReadableStore[UserId, CandidateTweetsList] = { buildOfflineCandidateStore( serviceIdentifier, datasetName = "offline_tweet_recommendations_from_interestedin_2020" ) } @Provides @Singleton @Named(ModuleNames.OfflineTweet2020Hl0El15CandidateStore) def offlineTweet2020Hl0El15CandidateMhStore( serviceIdentifier: ServiceIdentifier ): ReadableStore[UserId, CandidateTweetsList] = { buildOfflineCandidateStore( serviceIdentifier, datasetName = "offline_tweet_recommendations_from_interestedin_2020_hl_0_el_15" ) } @Provides @Singleton @Named(ModuleNames.OfflineTweet2020Hl2El15CandidateStore) def offlineTweet2020Hl2El15CandidateMhStore( serviceIdentifier: ServiceIdentifier ): ReadableStore[UserId, CandidateTweetsList] = { buildOfflineCandidateStore( serviceIdentifier, datasetName = "offline_tweet_recommendations_from_interestedin_2020_hl_2_el_15" ) } @Provides @Singleton @Named(ModuleNames.OfflineTweet2020Hl2El50CandidateStore) def offlineTweet2020Hl2El50CandidateMhStore( serviceIdentifier: ServiceIdentifier ): ReadableStore[UserId, CandidateTweetsList] = { buildOfflineCandidateStore( serviceIdentifier, datasetName = "offline_tweet_recommendations_from_interestedin_2020_hl_2_el_50" ) } @Provides @Singleton @Named(ModuleNames.OfflineTweet2020Hl8El50CandidateStore) def offlineTweet2020Hl8El50CandidateMhStore( serviceIdentifier: ServiceIdentifier ): ReadableStore[UserId, CandidateTweetsList] = { buildOfflineCandidateStore( serviceIdentifier, datasetName = "offline_tweet_recommendations_from_interestedin_2020_hl_8_el_50" ) } @Provides @Singleton @Named(ModuleNames.OfflineTweetMTSCandidateStore) def offlineTweetMTSCandidateMhStore( serviceIdentifier: ServiceIdentifier ): ReadableStore[UserId, CandidateTweetsList] = { buildOfflineCandidateStore( serviceIdentifier, datasetName = "offline_tweet_recommendations_from_mts_consumer_embeddings" ) } @Provides @Singleton @Named(ModuleNames.OfflineFavDecayedSumCandidateStore) def offlineFavDecayedSumCandidateStore( serviceIdentifier: ServiceIdentifier ): ReadableStore[UserId, CandidateTweetsList] = { buildOfflineCandidateStore( serviceIdentifier, datasetName = "offline_tweet_recommendations_from_decayed_sum" ) } @Provides @Singleton @Named(ModuleNames.OfflineFtrAt5Pop1000RankDecay11CandidateStore) def offlineFtrAt5Pop1000RankDecay11CandidateStore( serviceIdentifier: ServiceIdentifier ): ReadableStore[UserId, CandidateTweetsList] = { buildOfflineCandidateStore( serviceIdentifier, datasetName = "offline_tweet_recommendations_from_ftrat5_pop1000_rank_decay_1_1" ) } @Provides @Singleton @Named(ModuleNames.OfflineFtrAt5Pop10000RankDecay11CandidateStore) def offlineFtrAt5Pop10000RankDecay11CandidateStore( serviceIdentifier: ServiceIdentifier ): ReadableStore[UserId, CandidateTweetsList] = { buildOfflineCandidateStore( serviceIdentifier, datasetName = "offline_tweet_recommendations_from_ftrat5_pop10000_rank_decay_1_1" ) } private def buildOfflineCandidateStore( serviceIdentifier: ServiceIdentifier, datasetName: String ): ReadableStore[UserId, CandidateTweetsList] = { ManhattanRO .getReadableStoreWithMtls[Long, CandidateTweetsList]( ManhattanROConfig( HDFSPath(""), // not needed ApplicationID("multi_type_simclusters"), DatasetName(datasetName), Apollo ), ManhattanKVClientMtlsParams(serviceIdentifier) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RealGraphOonStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.twitter.app.Flag import com.twitter.cr_mixer.model.ModuleNames import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.store.strato.StratoFetchableStore import com.twitter.hermit.store.common.ObservedReadableStore import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.common.UserId import com.twitter.storehaus.ReadableStore import javax.inject.Named import javax.inject.Singleton import com.twitter.strato.client.{Client => StratoClient} import com.twitter.wtf.candidate.thriftscala.CandidateSeq object RealGraphOonStoreModule extends TwitterModule { private val userRealGraphOonColumnPath: Flag[String] = flag[String]( name = "crMixer.userRealGraphOonColumnPath", default = "recommendations/twistly/userRealgraphOon", help = "Strato column path for user real graph OON Store" ) @Provides @Singleton @Named(ModuleNames.RealGraphOonStore) def providesRealGraphOonStore( stratoClient: StratoClient, statsReceiver: StatsReceiver ): ReadableStore[UserId, CandidateSeq] = { val realGraphOonStratoFetchableStore = StratoFetchableStore .withUnitView[UserId, CandidateSeq](stratoClient, userRealGraphOonColumnPath()) ObservedReadableStore( realGraphOonStratoFetchableStore )(statsReceiver.scope("user_real_graph_oon_store")) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RealGraphStoreMhModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.google.inject.name.Named import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.common.UserId import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams import com.twitter.storehaus.ReadableStore import com.twitter.storehaus_internal.manhattan.Apollo import com.twitter.storehaus_internal.manhattan.ManhattanRO import com.twitter.storehaus_internal.manhattan.ManhattanROConfig import com.twitter.storehaus_internal.util.ApplicationID import com.twitter.storehaus_internal.util.DatasetName import com.twitter.storehaus_internal.util.HDFSPath import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.cr_mixer.param.decider.DeciderKey import com.twitter.hermit.store.common.DeciderableReadableStore import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.wtf.candidate.thriftscala.CandidateSeq object RealGraphStoreMhModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.RealGraphInStore) def providesRealGraphStoreMh( decider: CrMixerDecider, statsReceiver: StatsReceiver, manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, ): ReadableStore[UserId, CandidateSeq] = { implicit val valueCodec = new BinaryScalaCodec(CandidateSeq) val underlyingStore = ManhattanRO .getReadableStoreWithMtls[UserId, CandidateSeq]( ManhattanROConfig( HDFSPath(""), ApplicationID("cr_mixer_apollo"), DatasetName("real_graph_scores_apollo"), Apollo), manhattanKVClientMtlsParams ) val memCachedStore = ObservedMemcachedReadableStore .fromCacheClient( backingStore = underlyingStore, cacheClient = crMixerUnifiedCacheClient, ttl = 24.hours, )( valueInjection = valueCodec, statsReceiver = statsReceiver.scope("memCachedUserRealGraphMh"), keyToString = { k: UserId => s"uRGraph/$k" } ) DeciderableReadableStore( memCachedStore, decider.deciderGateBuilder.idGate(DeciderKey.enableRealGraphMhStoreDeciderKey), statsReceiver.scope("RealGraphMh") ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RepresentationManagerModule.scala ================================================ package com.twitter.cr_mixer.module import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.common.SimClustersEmbedding import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.common.UserId import com.twitter.storehaus.ReadableStore import com.twitter.strato.client.{Client => StratoClient} import com.twitter.representation_manager.thriftscala.SimClustersEmbeddingView import com.twitter.simclusters_v2.thriftscala.EmbeddingType import com.twitter.simclusters_v2.thriftscala.ModelVersion import com.google.inject.Provides import com.google.inject.Singleton import javax.inject.Named import com.twitter.cr_mixer.model.ModuleNames import com.twitter.frigate.common.store.strato.StratoFetchableStore import com.twitter.hermit.store.common.ObservedReadableStore import com.twitter.simclusters_v2.thriftscala.{SimClustersEmbedding => ThriftSimClustersEmbedding} object RepresentationManagerModule extends TwitterModule { private val ColPathPrefix = "recommendations/representation_manager/" private val SimclustersTweetColPath = ColPathPrefix + "simClustersEmbedding.Tweet" private val SimclustersUserColPath = ColPathPrefix + "simClustersEmbedding.User" @Provides @Singleton @Named(ModuleNames.RmsTweetLogFavLongestL2EmbeddingStore) def providesRepresentationManagerTweetStore( statsReceiver: StatsReceiver, stratoClient: StratoClient, ): ReadableStore[TweetId, SimClustersEmbedding] = { ObservedReadableStore( StratoFetchableStore .withView[Long, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( stratoClient, SimclustersTweetColPath, SimClustersEmbeddingView( EmbeddingType.LogFavLongestL2EmbeddingTweet, ModelVersion.Model20m145k2020)) .mapValues(SimClustersEmbedding(_)))( statsReceiver.scope("rms_tweet_log_fav_longest_l2_store")) } @Provides @Singleton @Named(ModuleNames.RmsUserFavBasedProducerEmbeddingStore) def providesRepresentationManagerUserFavBasedProducerEmbeddingStore( statsReceiver: StatsReceiver, stratoClient: StratoClient, ): ReadableStore[UserId, SimClustersEmbedding] = { ObservedReadableStore( StratoFetchableStore .withView[Long, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( stratoClient, SimclustersUserColPath, SimClustersEmbeddingView( EmbeddingType.FavBasedProducer, ModelVersion.Model20m145k2020 ) ) .mapValues(SimClustersEmbedding(_)))( statsReceiver.scope("rms_user_fav_based_producer_store")) } @Provides @Singleton @Named(ModuleNames.RmsUserLogFavInterestedInEmbeddingStore) def providesRepresentationManagerUserLogFavConsumerEmbeddingStore( statsReceiver: StatsReceiver, stratoClient: StratoClient, ): ReadableStore[UserId, SimClustersEmbedding] = { ObservedReadableStore( StratoFetchableStore .withView[Long, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( stratoClient, SimclustersUserColPath, SimClustersEmbeddingView( EmbeddingType.LogFavBasedUserInterestedIn, ModelVersion.Model20m145k2020 ) ) .mapValues(SimClustersEmbedding(_)))( statsReceiver.scope("rms_user_log_fav_interestedin_store")) } @Provides @Singleton @Named(ModuleNames.RmsUserFollowInterestedInEmbeddingStore) def providesRepresentationManagerUserFollowInterestedInEmbeddingStore( statsReceiver: StatsReceiver, stratoClient: StratoClient, ): ReadableStore[UserId, SimClustersEmbedding] = { ObservedReadableStore( StratoFetchableStore .withView[Long, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( stratoClient, SimclustersUserColPath, SimClustersEmbeddingView( EmbeddingType.FollowBasedUserInterestedIn, ModelVersion.Model20m145k2020 ) ) .mapValues(SimClustersEmbedding(_)))( statsReceiver.scope("rms_user_follow_interestedin_store")) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RepresentationScorerModule.scala ================================================ package com.twitter.cr_mixer.module import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.thriftscala.EmbeddingType import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.simclusters_v2.thriftscala.ModelVersion import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.store.strato.StratoFetchableStore import com.twitter.simclusters_v2.common.UserId import com.twitter.simclusters_v2.common.TweetId import com.twitter.strato.client.{Client => StratoClient} import com.twitter.storehaus.ReadableStore import com.twitter.simclusters_v2.thriftscala.ScoringAlgorithm import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.hermit.store.common.ObservedReadableStore import javax.inject.Named import com.twitter.cr_mixer.model.ModuleNames import com.twitter.representationscorer.thriftscala.ListScoreId object RepresentationScorerModule extends TwitterModule { private val rsxColumnPath = "recommendations/representation_scorer/listScore" private final val SimClusterModelVersion = ModelVersion.Model20m145k2020 private final val TweetEmbeddingType = EmbeddingType.LogFavBasedTweet @Provides @Singleton @Named(ModuleNames.RsxStore) def providesRepresentationScorerStore( statsReceiver: StatsReceiver, stratoClient: StratoClient, ): ReadableStore[(UserId, TweetId), Double] = { ObservedReadableStore( StratoFetchableStore .withUnitView[ListScoreId, Double](stratoClient, rsxColumnPath).composeKeyMapping[( UserId, TweetId )] { key => representationScorerStoreKeyMapping(key._1, key._2) } )(statsReceiver.scope("rsx_store")) } private def representationScorerStoreKeyMapping(t1: TweetId, t2: TweetId): ListScoreId = { ListScoreId( algorithm = ScoringAlgorithm.PairEmbeddingLogCosineSimilarity, modelVersion = SimClusterModelVersion, targetEmbeddingType = TweetEmbeddingType, targetId = InternalId.TweetId(t1), candidateEmbeddingType = TweetEmbeddingType, candidateIds = Seq(InternalId.TweetId(t2)) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SampleSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.similarity_engine.LookupSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.common.UserId import com.twitter.storehaus.ReadableStore import javax.inject.Singleton /** * In this example we build a [[StandardSimilarityEngine]] to wrap a dummy store */ object SimpleSimilarityEngineModule extends TwitterModule { @Provides @Singleton def providesSimpleSimilarityEngine( timeoutConfig: TimeoutConfig, globalStats: StatsReceiver ): StandardSimilarityEngine[UserId, (TweetId, Double)] = { // Inject your readableStore implementation here val dummyStore = ReadableStore.fromMap( Map( 1L -> Seq((100L, 1.0), (101L, 1.0)), 2L -> Seq((200L, 2.0), (201L, 2.0)), 3L -> Seq((300L, 3.0), (301L, 3.0)) )) new StandardSimilarityEngine[UserId, (TweetId, Double)]( implementingStore = dummyStore, identifier = SimilarityEngineType.EnumUnknownSimilarityEngineType(9997), globalStats = globalStats, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = None, enableFeatureSwitch = None ) ) ) } } /** * In this example we build a [[LookupSimilarityEngine]] to wrap a dummy store with 2 versions */ object LookupSimilarityEngineModule extends TwitterModule { @Provides @Singleton def providesLookupSimilarityEngine( timeoutConfig: TimeoutConfig, globalStats: StatsReceiver ): LookupSimilarityEngine[UserId, (TweetId, Double)] = { // Inject your readableStore implementation here val dummyStoreV1 = ReadableStore.fromMap( Map( 1L -> Seq((100L, 1.0), (101L, 1.0)), 2L -> Seq((200L, 2.0), (201L, 2.0)), )) val dummyStoreV2 = ReadableStore.fromMap( Map( 1L -> Seq((100L, 1.0), (101L, 1.0)), 2L -> Seq((200L, 2.0), (201L, 2.0)), )) new LookupSimilarityEngine[UserId, (TweetId, Double)]( versionedStoreMap = Map( "V1" -> dummyStoreV1, "V2" -> dummyStoreV2 ), identifier = SimilarityEngineType.EnumUnknownSimilarityEngineType(9998), globalStats = globalStats, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = None, enableFeatureSwitch = None ) ) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SimClustersANNServiceNameToClientMapper.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.cr_mixer.model.ModuleNames import com.twitter.inject.TwitterModule import com.twitter.simclustersann.thriftscala.SimClustersANNService import javax.inject.Named object SimClustersANNServiceNameToClientMapper extends TwitterModule { @Provides @Singleton def providesSimClustersANNServiceNameToClientMapping( @Named(ModuleNames.ProdSimClustersANNServiceClientName) simClustersANNServiceProd: SimClustersANNService.MethodPerEndpoint, @Named(ModuleNames.ExperimentalSimClustersANNServiceClientName) simClustersANNServiceExperimental: SimClustersANNService.MethodPerEndpoint, @Named(ModuleNames.SimClustersANNServiceClientName1) simClustersANNService1: SimClustersANNService.MethodPerEndpoint, @Named(ModuleNames.SimClustersANNServiceClientName2) simClustersANNService2: SimClustersANNService.MethodPerEndpoint, @Named(ModuleNames.SimClustersANNServiceClientName3) simClustersANNService3: SimClustersANNService.MethodPerEndpoint, @Named(ModuleNames.SimClustersANNServiceClientName5) simClustersANNService5: SimClustersANNService.MethodPerEndpoint, @Named(ModuleNames.SimClustersANNServiceClientName4) simClustersANNService4: SimClustersANNService.MethodPerEndpoint ): Map[String, SimClustersANNService.MethodPerEndpoint] = { Map[String, SimClustersANNService.MethodPerEndpoint]( "simclusters-ann" -> simClustersANNServiceProd, "simclusters-ann-experimental" -> simClustersANNServiceExperimental, "simclusters-ann-1" -> simClustersANNService1, "simclusters-ann-2" -> simClustersANNService2, "simclusters-ann-3" -> simClustersANNService3, "simclusters-ann-5" -> simClustersANNService5, "simclusters-ann-4" -> simClustersANNService4 ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SkitStratoStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.google.inject.name.Named import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.keyHasher import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.store.strato.StratoFetchableStore import com.twitter.hermit.store.common.ObservedCachedReadableStore import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.hermit.store.common.ObservedReadableStore import com.twitter.inject.TwitterModule import com.twitter.relevance_platform.common.injection.LZ4Injection import com.twitter.relevance_platform.common.injection.SeqObjectInjection import com.twitter.storehaus.ReadableStore import com.twitter.strato.client.Client import com.twitter.topic_recos.thriftscala.TopicTopTweets import com.twitter.topic_recos.thriftscala.TopicTweet import com.twitter.topic_recos.thriftscala.TopicTweetPartitionFlatKey /** * Strato store that wraps the topic top tweets pipeline indexed from a Summingbird job */ object SkitStratoStoreModule extends TwitterModule { val column = "recommendations/topic_recos/topicTopTweets" @Provides @Singleton @Named(ModuleNames.SkitStratoStoreName) def providesSkitStratoStore( @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, stratoClient: Client, statsReceiver: StatsReceiver ): ReadableStore[TopicTweetPartitionFlatKey, Seq[TopicTweet]] = { val skitStore = ObservedReadableStore( StratoFetchableStore .withUnitView[TopicTweetPartitionFlatKey, TopicTopTweets](stratoClient, column))( statsReceiver.scope(ModuleNames.SkitStratoStoreName)).mapValues { topicTopTweets => topicTopTweets.topTweets } val memCachedStore = ObservedMemcachedReadableStore .fromCacheClient( backingStore = skitStore, cacheClient = crMixerUnifiedCacheClient, ttl = 10.minutes )( valueInjection = LZ4Injection.compose(SeqObjectInjection[TopicTweet]()), statsReceiver = statsReceiver.scope("memcached_skit_store"), keyToString = { k => s"skit:${keyHasher.hashKey(k.toString.getBytes)}" } ) ObservedCachedReadableStore.from[TopicTweetPartitionFlatKey, Seq[TopicTweet]]( memCachedStore, ttl = 5.minutes, maxKeys = 100000, // ~150MB max cacheName = "skit_in_memory_cache", windowSize = 10000L )(statsReceiver.scope("skit_in_memory_cache")) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/StrongTiePredictionStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.app.Flag import com.twitter.cr_mixer.model.ModuleNames import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.store.strato.StratoFetchableStore import com.twitter.hermit.store.common.ObservedReadableStore import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.common.UserId import com.twitter.hermit.stp.thriftscala.STPResult import com.twitter.storehaus.ReadableStore import com.twitter.strato.client.{Client => StratoClient} import javax.inject.Named object StrongTiePredictionStoreModule extends TwitterModule { private val strongTiePredictionColumnPath: Flag[String] = flag[String]( name = "crMixer.strongTiePredictionColumnPath", default = "onboarding/userrecs/strong_tie_prediction_big", help = "Strato column path for StrongTiePredictionStore" ) @Provides @Singleton @Named(ModuleNames.StpStore) def providesStrongTiePredictionStore( statsReceiver: StatsReceiver, stratoClient: StratoClient, ): ReadableStore[UserId, STPResult] = { val strongTiePredictionStratoFetchableStore = StratoFetchableStore .withUnitView[UserId, STPResult](stratoClient, strongTiePredictionColumnPath()) ObservedReadableStore( strongTiePredictionStratoFetchableStore )(statsReceiver.scope("strong_tie_prediction_big_store")) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TripCandidateStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.twitter.cr_mixer.model.ModuleNames import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.store.strato.StratoFetchableStore import com.twitter.hermit.store.common.ObservedReadableStore import com.twitter.inject.TwitterModule import com.twitter.storehaus.ReadableStore import com.twitter.strato.client.{Client => StratoClient} import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweet import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain import javax.inject.Named object TripCandidateStoreModule extends TwitterModule { private val stratoColumn = "trends/trip/tripTweetsDataflowProd" @Provides @Named(ModuleNames.TripCandidateStore) def providesSimClustersTripCandidateStore( statsReceiver: StatsReceiver, stratoClient: StratoClient ): ReadableStore[TripDomain, Seq[TripTweet]] = { val tripCandidateStratoFetchableStore = StratoFetchableStore .withUnitView[TripDomain, TripTweets](stratoClient, stratoColumn) .mapValues(_.tweets) ObservedReadableStore( tripCandidateStratoFetchableStore )(statsReceiver.scope("simclusters_trip_candidate_store")) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetInfoStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Module import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.contentrecommender.thriftscala.TweetInfo import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.frigate.common.store.health.TweetHealthModelStore import com.twitter.frigate.common.store.health.TweetHealthModelStore.TweetHealthModelStoreConfig import com.twitter.frigate.common.store.health.UserHealthModelStore import com.twitter.frigate.thriftscala.TweetHealthScores import com.twitter.frigate.thriftscala.UserAgathaScores import com.twitter.hermit.store.common.DeciderableReadableStore import com.twitter.hermit.store.common.ObservedCachedReadableStore import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.hermit.store.common.ObservedReadableStore import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.common.UserId import com.twitter.storehaus.ReadableStore import com.twitter.strato.client.{Client => StratoClient} import com.twitter.contentrecommender.store.TweetInfoStore import com.twitter.contentrecommender.store.TweetyPieFieldsStore import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderKey import com.twitter.frigate.data_pipeline.scalding.thriftscala.BlueVerifiedAnnotationsV2 import com.twitter.recos.user_tweet_graph_plus.thriftscala.UserTweetGraphPlus import com.twitter.recos.user_tweet_graph_plus.thriftscala.TweetEngagementScores import com.twitter.relevance_platform.common.health_store.UserMediaRepresentationHealthStore import com.twitter.relevance_platform.common.health_store.MagicRecsRealTimeAggregatesStore import com.twitter.relevance_platform.thriftscala.MagicRecsRealTimeAggregatesScores import com.twitter.relevance_platform.thriftscala.UserMediaRepresentationScores import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams import com.twitter.tweetypie.thriftscala.TweetService import com.twitter.util.Future import com.twitter.util.JavaTimer import com.twitter.util.Timer import javax.inject.Named object TweetInfoStoreModule extends TwitterModule { implicit val timer: Timer = new JavaTimer(true) override def modules: Seq[Module] = Seq(UnifiedCacheClient) @Provides @Singleton def providesTweetInfoStore( statsReceiver: StatsReceiver, serviceIdentifier: ServiceIdentifier, stratoClient: StratoClient, @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, tweetyPieService: TweetService.MethodPerEndpoint, userTweetGraphPlusService: UserTweetGraphPlus.MethodPerEndpoint, @Named(ModuleNames.BlueVerifiedAnnotationStore) blueVerifiedAnnotationStore: ReadableStore[ String, BlueVerifiedAnnotationsV2 ], decider: CrMixerDecider ): ReadableStore[TweetId, TweetInfo] = { val tweetEngagementScoreStore: ReadableStore[TweetId, TweetEngagementScores] = { val underlyingStore = ObservedReadableStore(new ReadableStore[TweetId, TweetEngagementScores] { override def get( k: TweetId ): Future[Option[TweetEngagementScores]] = { userTweetGraphPlusService.tweetEngagementScore(k).map { Some(_) } } })(statsReceiver.scope("UserTweetGraphTweetEngagementScoreStore")) DeciderableReadableStore( underlyingStore, decider.deciderGateBuilder.idGate( DeciderKey.enableUtgRealTimeTweetEngagementScoreDeciderKey), statsReceiver.scope("UserTweetGraphTweetEngagementScoreStore") ) } val tweetHealthModelStore: ReadableStore[TweetId, TweetHealthScores] = { val underlyingStore = TweetHealthModelStore.buildReadableStore( stratoClient, Some( TweetHealthModelStoreConfig( enablePBlock = true, enableToxicity = true, enablePSpammy = true, enablePReported = true, enableSpammyTweetContent = true, enablePNegMultimodal = true, )) )(statsReceiver.scope("UnderlyingTweetHealthModelStore")) DeciderableReadableStore( ObservedMemcachedReadableStore.fromCacheClient( backingStore = underlyingStore, cacheClient = crMixerUnifiedCacheClient, ttl = 2.hours )( valueInjection = BinaryScalaCodec(TweetHealthScores), statsReceiver = statsReceiver.scope("memCachedTweetHealthModelStore"), keyToString = { k: TweetId => s"tHMS/$k" } ), decider.deciderGateBuilder.idGate(DeciderKey.enableHealthSignalsScoreDeciderKey), statsReceiver.scope("TweetHealthModelStore") ) // use s"tHMS/$k" instead of s"tweetHealthModelStore/$k" to differentiate from CR cache } val userHealthModelStore: ReadableStore[UserId, UserAgathaScores] = { val underlyingStore = UserHealthModelStore.buildReadableStore(stratoClient)( statsReceiver.scope("UnderlyingUserHealthModelStore")) DeciderableReadableStore( ObservedMemcachedReadableStore.fromCacheClient( backingStore = underlyingStore, cacheClient = crMixerUnifiedCacheClient, ttl = 18.hours )( valueInjection = BinaryScalaCodec(UserAgathaScores), statsReceiver = statsReceiver.scope("memCachedUserHealthModelStore"), keyToString = { k: UserId => s"uHMS/$k" } ), decider.deciderGateBuilder.idGate(DeciderKey.enableUserAgathaScoreDeciderKey), statsReceiver.scope("UserHealthModelStore") ) } val userMediaRepresentationHealthStore: ReadableStore[UserId, UserMediaRepresentationScores] = { val underlyingStore = UserMediaRepresentationHealthStore.buildReadableStore( manhattanKVClientMtlsParams, statsReceiver.scope("UnderlyingUserMediaRepresentationHealthStore") ) DeciderableReadableStore( ObservedMemcachedReadableStore.fromCacheClient( backingStore = underlyingStore, cacheClient = crMixerUnifiedCacheClient, ttl = 12.hours )( valueInjection = BinaryScalaCodec(UserMediaRepresentationScores), statsReceiver = statsReceiver.scope("memCacheUserMediaRepresentationHealthStore"), keyToString = { k: UserId => s"uMRHS/$k" } ), decider.deciderGateBuilder.idGate(DeciderKey.enableUserMediaRepresentationStoreDeciderKey), statsReceiver.scope("UserMediaRepresentationHealthStore") ) } val magicRecsRealTimeAggregatesStore: ReadableStore[ TweetId, MagicRecsRealTimeAggregatesScores ] = { val underlyingStore = MagicRecsRealTimeAggregatesStore.buildReadableStore( serviceIdentifier, statsReceiver.scope("UnderlyingMagicRecsRealTimeAggregatesScores") ) DeciderableReadableStore( underlyingStore, decider.deciderGateBuilder.idGate(DeciderKey.enableMagicRecsRealTimeAggregatesStore), statsReceiver.scope("MagicRecsRealTimeAggregatesStore") ) } val tweetInfoStore: ReadableStore[TweetId, TweetInfo] = { val underlyingStore = TweetInfoStore( TweetyPieFieldsStore.getStoreFromTweetyPie(tweetyPieService), userMediaRepresentationHealthStore, magicRecsRealTimeAggregatesStore, tweetEngagementScoreStore, blueVerifiedAnnotationStore )(statsReceiver.scope("tweetInfoStore")) val memcachedStore = ObservedMemcachedReadableStore.fromCacheClient( backingStore = underlyingStore, cacheClient = crMixerUnifiedCacheClient, ttl = 15.minutes, // Hydrating tweetInfo is now a required step for all candidates, // hence we needed to tune these thresholds. asyncUpdate = serviceIdentifier.environment == "prod" )( valueInjection = BinaryScalaCodec(TweetInfo), statsReceiver = statsReceiver.scope("memCachedTweetInfoStore"), keyToString = { k: TweetId => s"tIS/$k" } ) ObservedCachedReadableStore.from( memcachedStore, ttl = 15.minutes, maxKeys = 8388607, // Check TweetInfo definition. size~92b. Around 736 MB windowSize = 10000L, cacheName = "tweet_info_cache", maxMultiGetSize = 20 )(statsReceiver.scope("inMemoryCachedTweetInfoStore")) } tweetInfoStore } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetRecentEngagedUserStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.app.Flag import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.store.strato.StratoFetchableStore import com.twitter.hermit.store.common.ObservedReadableStore import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.common.TweetId import com.twitter.storehaus.ReadableStore import com.twitter.strato.client.{Client => StratoClient} import com.twitter.twistly.thriftscala.TweetRecentEngagedUsers object TweetRecentEngagedUserStoreModule extends TwitterModule { private val tweetRecentEngagedUsersStoreDefaultVersion = 0 // DefaultVersion for tweetEngagedUsersStore, whose key = (tweetId, DefaultVersion) private val tweetRecentEngagedUsersColumnPath: Flag[String] = flag[String]( name = "crMixer.tweetRecentEngagedUsersColumnPath", default = "recommendations/twistly/tweetRecentEngagedUsers", help = "Strato column path for TweetRecentEngagedUsersStore" ) private type Version = Long @Provides @Singleton def providesTweetRecentEngagedUserStore( statsReceiver: StatsReceiver, stratoClient: StratoClient, ): ReadableStore[TweetId, TweetRecentEngagedUsers] = { val tweetRecentEngagedUsersStratoFetchableStore = StratoFetchableStore .withUnitView[(TweetId, Version), TweetRecentEngagedUsers]( stratoClient, tweetRecentEngagedUsersColumnPath()).composeKeyMapping[TweetId](tweetId => (tweetId, tweetRecentEngagedUsersStoreDefaultVersion)) ObservedReadableStore( tweetRecentEngagedUsersStratoFetchableStore )(statsReceiver.scope("tweet_recent_engaged_users_store")) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetRecommendationResultsStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.thriftscala.CrMixerTweetResponse import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.hermit.store.common.ReadableWritableStore import com.twitter.hermit.store.common.ObservedReadableWritableMemcacheStore import com.twitter.simclusters_v2.common.UserId import javax.inject.Named object TweetRecommendationResultsStoreModule extends TwitterModule { @Provides @Singleton def providesTweetRecommendationResultsStore( @Named(ModuleNames.TweetRecommendationResultsCache) tweetRecommendationResultsCacheClient: MemcachedClient, statsReceiver: StatsReceiver ): ReadableWritableStore[UserId, CrMixerTweetResponse] = { ObservedReadableWritableMemcacheStore.fromCacheClient( cacheClient = tweetRecommendationResultsCacheClient, ttl = 24.hours)( valueInjection = BinaryScalaCodec(CrMixerTweetResponse), statsReceiver = statsReceiver.scope("TweetRecommendationResultsMemcacheStore"), keyToString = { k: UserId => k.toString } ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TwhinCollabFilterStratoStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.inject.TwitterModule import com.twitter.cr_mixer.model.ModuleNames import com.twitter.frigate.common.store.strato.StratoFetchableStore import com.twitter.cr_mixer.similarity_engine.TwhinCollabFilterSimilarityEngine.TwhinCollabFilterView import com.twitter.strato.client.{Client => StratoClient} import com.twitter.simclusters_v2.common.TweetId import com.twitter.storehaus.ReadableStore import javax.inject.Named object TwhinCollabFilterStratoStoreModule extends TwitterModule { val stratoColumnPath: String = "cuad/twhin/getCollabFilterTweetCandidatesProd.User" @Provides @Singleton @Named(ModuleNames.TwhinCollabFilterStratoStoreForFollow) def providesTwhinCollabFilterStratoStoreForFollow( stratoClient: StratoClient ): ReadableStore[Long, Seq[TweetId]] = { StratoFetchableStore.withView[Long, TwhinCollabFilterView, Seq[TweetId]]( stratoClient, column = stratoColumnPath, view = TwhinCollabFilterView("follow_2022_03_10_c_500K") ) } @Provides @Singleton @Named(ModuleNames.TwhinCollabFilterStratoStoreForEngagement) def providesTwhinCollabFilterStratoStoreForEngagement( stratoClient: StratoClient ): ReadableStore[Long, Seq[TweetId]] = { StratoFetchableStore.withView[Long, TwhinCollabFilterView, Seq[TweetId]]( stratoClient, column = stratoColumnPath, view = TwhinCollabFilterView("engagement_2022_04_10_c_500K")) } @Provides @Singleton @Named(ModuleNames.TwhinMultiClusterStratoStoreForFollow) def providesTwhinMultiClusterStratoStoreForFollow( stratoClient: StratoClient ): ReadableStore[Long, Seq[TweetId]] = { StratoFetchableStore.withView[Long, TwhinCollabFilterView, Seq[TweetId]]( stratoClient, column = stratoColumnPath, view = TwhinCollabFilterView("multiclusterFollow20220921") ) } @Provides @Singleton @Named(ModuleNames.TwhinMultiClusterStratoStoreForEngagement) def providesTwhinMultiClusterStratoStoreForEngagement( stratoClient: StratoClient ): ReadableStore[Long, Seq[TweetId]] = { StratoFetchableStore.withView[Long, TwhinCollabFilterView, Seq[TweetId]]( stratoClient, column = stratoColumnPath, view = TwhinCollabFilterView("multiclusterEng20220921")) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TwiceClustersMembersStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.app.Flag import com.twitter.cr_mixer.model.ModuleNames import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.store.strato.StratoFetchableStore import com.twitter.hermit.store.common.ObservedReadableStore import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.common.UserId import com.twitter.storehaus.ReadableStore import com.twitter.strato.client.{Client => StratoClient} import com.twitter.simclusters_v2.thriftscala.OrderedClustersAndMembers import javax.inject.Named object TwiceClustersMembersStoreModule extends TwitterModule { private val twiceClustersMembersColumnPath: Flag[String] = flag[String]( name = "crMixer.twiceClustersMembersColumnPath", default = "recommendations/simclusters_v2/embeddings/TwiceClustersMembersLargestDimApeSimilarity", help = "Strato column path for TweetRecentEngagedUsersStore" ) @Provides @Singleton @Named(ModuleNames.TwiceClustersMembersStore) def providesTweetRecentEngagedUserStore( statsReceiver: StatsReceiver, stratoClient: StratoClient, ): ReadableStore[UserId, OrderedClustersAndMembers] = { val twiceClustersMembersStratoFetchableStore = StratoFetchableStore .withUnitView[UserId, OrderedClustersAndMembers]( stratoClient, twiceClustersMembersColumnPath()) ObservedReadableStore( twiceClustersMembersStratoFetchableStore )(statsReceiver.scope("twice_clusters_members_largestDimApe_similarity_store")) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UnifiedCacheClient.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.app.Flag import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.model.ModuleNames import com.twitter.finagle.memcached.Client import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.storehaus_internal.memcache.MemcacheStore import com.twitter.storehaus_internal.util.ClientName import com.twitter.storehaus_internal.util.ZkEndPoint import javax.inject.Named object UnifiedCacheClient extends TwitterModule { private val TIME_OUT = 20.milliseconds val crMixerUnifiedCacheDest: Flag[String] = flag[String]( name = "crMixer.unifiedCacheDest", default = "/s/cache/content_recommender_unified_v2", help = "Wily path to Content Recommender unified cache" ) val tweetRecommendationResultsCacheDest: Flag[String] = flag[String]( name = "tweetRecommendationResults.CacheDest", default = "/s/cache/tweet_recommendation_results", help = "Wily path to CrMixer getTweetRecommendations() results cache" ) val earlybirdTweetsCacheDest: Flag[String] = flag[String]( name = "earlybirdTweets.CacheDest", default = "/s/cache/crmixer_earlybird_tweets", help = "Wily path to CrMixer Earlybird Recency Based Similarity Engine result cache" ) @Provides @Singleton @Named(ModuleNames.UnifiedCache) def provideUnifiedCacheClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver, ): Client = MemcacheStore.memcachedClient( name = ClientName("memcache-content-recommender-unified"), dest = ZkEndPoint(crMixerUnifiedCacheDest()), statsReceiver = statsReceiver.scope("cache_client"), serviceIdentifier = serviceIdentifier, timeout = TIME_OUT ) @Provides @Singleton @Named(ModuleNames.TweetRecommendationResultsCache) def providesTweetRecommendationResultsCache( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver, ): Client = MemcacheStore.memcachedClient( name = ClientName("memcache-tweet-recommendation-results"), dest = ZkEndPoint(tweetRecommendationResultsCacheDest()), statsReceiver = statsReceiver.scope("cache_client"), serviceIdentifier = serviceIdentifier, timeout = TIME_OUT ) @Provides @Singleton @Named(ModuleNames.EarlybirdTweetsCache) def providesEarlybirdTweetsCache( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver, ): Client = MemcacheStore.memcachedClient( name = ClientName("memcache-crmixer-earlybird-tweets"), dest = ZkEndPoint(earlybirdTweetsCacheDest()), statsReceiver = statsReceiver.scope("cache_client"), serviceIdentifier = serviceIdentifier, timeout = TIME_OUT ) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserSignalServiceColumnModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.storehaus.ReadableStore import com.twitter.strato.client.{Client => StratoClient} import com.twitter.cr_mixer.model.ModuleNames import com.twitter.frigate.common.store.strato.StratoFetchableStore import com.twitter.hermit.store.common.ObservedReadableStore import com.twitter.usersignalservice.thriftscala.BatchSignalRequest import com.twitter.usersignalservice.thriftscala.BatchSignalResponse import javax.inject.Named object UserSignalServiceColumnModule extends TwitterModule { private val UssColumnPath = "recommendations/user-signal-service/signals" @Provides @Singleton @Named(ModuleNames.UssStratoColumn) def providesUserSignalServiceStore( statsReceiver: StatsReceiver, stratoClient: StratoClient, ): ReadableStore[BatchSignalRequest, BatchSignalResponse] = { ObservedReadableStore( StratoFetchableStore .withUnitView[BatchSignalRequest, BatchSignalResponse](stratoClient, UssColumnPath))( statsReceiver.scope("user_signal_service_store")) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserSignalServiceStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.storehaus.ReadableStore import com.twitter.strato.client.{Client => StratoClient} import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.source_signal.UssStore import com.twitter.cr_mixer.source_signal.UssStore.Query import com.twitter.frigate.common.store.strato.StratoFetchableStore import com.twitter.hermit.store.common.ObservedReadableStore import com.twitter.usersignalservice.thriftscala.BatchSignalRequest import com.twitter.usersignalservice.thriftscala.BatchSignalResponse import com.twitter.usersignalservice.thriftscala.SignalType import com.twitter.usersignalservice.thriftscala.{Signal => UssSignal} import javax.inject.Named object UserSignalServiceStoreModule extends TwitterModule { private val UssColumnPath = "recommendations/user-signal-service/signals" @Provides @Singleton @Named(ModuleNames.UssStore) def providesUserSignalServiceStore( statsReceiver: StatsReceiver, stratoClient: StratoClient, ): ReadableStore[Query, Seq[(SignalType, Seq[UssSignal])]] = { ObservedReadableStore( UssStore( StratoFetchableStore .withUnitView[BatchSignalRequest, BatchSignalResponse](stratoClient, UssColumnPath), statsReceiver))(statsReceiver.scope("user_signal_service_store")) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserStateStoreModule.scala ================================================ package com.twitter.cr_mixer.module import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.bijection.Bufferable import com.twitter.bijection.Injection import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.cr_mixer.model.ModuleNames import com.twitter.conversions.DurationOps._ import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.finagle.stats.StatsReceiver import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.common.UserId import com.twitter.snowflake.id.SnowflakeId import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams import com.twitter.storehaus.ReadableStore import com.twitter.storehaus_internal.manhattan.ManhattanRO import com.twitter.storehaus_internal.manhattan.ManhattanROConfig import com.twitter.storehaus_internal.util.HDFSPath import com.twitter.core_workflows.user_model.thriftscala.UserState import com.twitter.core_workflows.user_model.thriftscala.CondensedUserState import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderKey import com.twitter.hermit.store.common.DeciderableReadableStore import com.twitter.storehaus_internal.manhattan.Apollo import com.twitter.storehaus_internal.util.ApplicationID import com.twitter.storehaus_internal.util.DatasetName import com.twitter.util.Duration import com.twitter.util.Future import com.twitter.util.JavaTimer import com.twitter.util.Time import com.twitter.util.TimeoutException import com.twitter.util.Timer import javax.inject.Named object UserStateStoreModule extends TwitterModule { implicit val timer: Timer = new JavaTimer(true) final val NewUserCreateDaysThreshold = 7 final val DefaultUnknownUserStateValue = 100 // Convert CondensedUserState to UserState Enum // If CondensedUserState is None, back fill by checking whether the user is new user class UserStateStore( userStateStore: ReadableStore[UserId, CondensedUserState], timeout: Duration, statsReceiver: StatsReceiver) extends ReadableStore[UserId, UserState] { override def get(userId: UserId): Future[Option[UserState]] = { userStateStore .get(userId).map(_.flatMap(_.userState)).map { case Some(userState) => Some(userState) case None => val isNewUser = SnowflakeId.timeFromIdOpt(userId).exists { userCreateTime => Time.now - userCreateTime < Duration.fromDays(NewUserCreateDaysThreshold) } if (isNewUser) Some(UserState.New) else Some(UserState.EnumUnknownUserState(DefaultUnknownUserStateValue)) }.raiseWithin(timeout)(timer).rescue { case _: TimeoutException => statsReceiver.counter("TimeoutException").incr() Future.None } } } @Provides @Singleton def providesUserStateStore( crMixerDecider: CrMixerDecider, statsReceiver: StatsReceiver, manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, timeoutConfig: TimeoutConfig ): ReadableStore[UserId, UserState] = { val underlyingStore = new UserStateStore( ManhattanRO .getReadableStoreWithMtls[UserId, CondensedUserState]( ManhattanROConfig( HDFSPath(""), ApplicationID("cr_mixer_apollo"), DatasetName("condensed_user_state"), Apollo), manhattanKVClientMtlsParams )( implicitly[Injection[Long, Array[Byte]]], BinaryScalaCodec(CondensedUserState) ), timeoutConfig.userStateStoreTimeout, statsReceiver.scope("UserStateStore") ).mapValues(_.value) // Read the value of Enum so that we only caches the Int val memCachedStore = ObservedMemcachedReadableStore .fromCacheClient( backingStore = underlyingStore, cacheClient = crMixerUnifiedCacheClient, ttl = 24.hours, )( valueInjection = Bufferable.injectionOf[Int], // Cache Value is Enum Value for UserState statsReceiver = statsReceiver.scope("memCachedUserStateStore"), keyToString = { k: UserId => s"uState/$k" } ).mapValues(value => UserState.getOrUnknown(value)) DeciderableReadableStore( memCachedStore, crMixerDecider.deciderGateBuilder.idGate(DeciderKey.enableUserStateStoreDeciderKey), statsReceiver.scope("UserStateStore") ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/ABDeciderModule.scala ================================================ package com.twitter.cr_mixer.module.core import com.google.inject.Provides import com.google.inject.name.Named import com.twitter.abdecider.ABDeciderFactory import com.twitter.abdecider.LoggingABDecider import com.twitter.cr_mixer.model.ModuleNames import com.twitter.inject.TwitterModule import com.twitter.inject.annotations.Flag import com.twitter.logging.Logger import javax.inject.Singleton object ABDeciderModule extends TwitterModule { flag( name = "abdecider.path", default = "/usr/local/config/abdecider/abdecider.yml", help = "path to the abdecider Yml file location" ) @Provides @Singleton def provideABDecider( @Flag("abdecider.path") abDeciderYmlPath: String, @Named(ModuleNames.AbDeciderLogger) scribeLogger: Logger ): LoggingABDecider = { ABDeciderFactory( abDeciderYmlPath = abDeciderYmlPath, scribeLogger = Some(scribeLogger), environment = Some("production") ).buildWithLogging() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/CrMixerFlagModule.scala ================================================ package com.twitter.cr_mixer.module.core import com.twitter.inject.TwitterModule object CrMixerFlagName { val SERVICE_FLAG = "cr_mixer.flag" val DarkTrafficFilterDeciderKey = "thrift.dark.traffic.filter.decider_key" } object CrMixerFlagModule extends TwitterModule { import CrMixerFlagName._ flag[Boolean](name = SERVICE_FLAG, default = false, help = "This is a CR Mixer flag") flag[String]( name = DarkTrafficFilterDeciderKey, default = "dark_traffic_filter", help = "Dark traffic filter decider key" ) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/CrMixerLoggingABDeciderModule.scala ================================================ package com.twitter.cr_mixer.module.core import com.google.inject.Provides import com.twitter.abdecider.LoggingABDecider import com.twitter.cr_mixer.featureswitch.CrMixerLoggingABDecider import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import javax.inject.Singleton object CrMixerLoggingABDeciderModule extends TwitterModule { @Provides @Singleton def provideABDecider( loggingABDecider: LoggingABDecider, statsReceiver: StatsReceiver ): CrMixerLoggingABDecider = { CrMixerLoggingABDecider(loggingABDecider, statsReceiver) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/FeatureContextBuilderModule.scala ================================================ package com.twitter.cr_mixer.module.core import com.google.inject.Provides import com.twitter.discovery.common.configapi.FeatureContextBuilder import com.twitter.featureswitches.v2.FeatureSwitches import com.twitter.inject.TwitterModule import javax.inject.Singleton object FeatureContextBuilderModule extends TwitterModule { @Provides @Singleton def providesFeatureContextBuilder(featureSwitches: FeatureSwitches): FeatureContextBuilder = { FeatureContextBuilder(featureSwitches) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/FeatureSwitchesModule.scala ================================================ package com.twitter.cr_mixer.module.core import com.google.inject.Provides import com.twitter.cr_mixer.featureswitch.CrMixerLoggingABDecider import com.twitter.featureswitches.v2.FeatureSwitches import com.twitter.featureswitches.v2.builder.FeatureSwitchesBuilder import com.twitter.featureswitches.v2.experimentation.NullBucketImpressor import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.inject.annotations.Flag import com.twitter.util.Duration import javax.inject.Singleton object FeatureSwitchesModule extends TwitterModule { flag( name = "featureswitches.path", default = "/features/cr-mixer/main", help = "path to the featureswitch configuration directory" ) flag( "use_config_repo_mirror.bool", false, "If true, read config from a different directory, to facilitate testing.") val DefaultFastRefresh: Boolean = false val AddServiceDetailsFromAurora: Boolean = true val ImpressExperiments: Boolean = true @Provides @Singleton def providesFeatureSwitches( @Flag("featureswitches.path") featureSwitchDirectory: String, @Flag("use_config_repo_mirror.bool") useConfigRepoMirrorFlag: Boolean, abDecider: CrMixerLoggingABDecider, statsReceiver: StatsReceiver ): FeatureSwitches = { val configRepoAbsPath = getConfigRepoAbsPath(useConfigRepoMirrorFlag) val fastRefresh = shouldFastRefresh(useConfigRepoMirrorFlag) val featureSwitches = FeatureSwitchesBuilder() .abDecider(abDecider) .statsReceiver(statsReceiver.scope("featureswitches-v2")) .configRepoAbsPath(configRepoAbsPath) .featuresDirectory(featureSwitchDirectory) .limitToReferencedExperiments(shouldLimit = true) .experimentImpressionStatsEnabled(true) if (!ImpressExperiments) featureSwitches.experimentBucketImpressor(NullBucketImpressor) if (AddServiceDetailsFromAurora) featureSwitches.serviceDetailsFromAurora() if (fastRefresh) featureSwitches.refreshPeriod(Duration.fromSeconds(10)) featureSwitches.build() } private def getConfigRepoAbsPath( useConfigRepoMirrorFlag: Boolean ): String = { if (useConfigRepoMirrorFlag) "config_repo_mirror/" else "/usr/local/config" } private def shouldFastRefresh( useConfigRepoMirrorFlag: Boolean ): Boolean = { if (useConfigRepoMirrorFlag) true else DefaultFastRefresh } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/KafkaProducerModule.scala ================================================ package com.twitter.cr_mixer.module.core import com.google.inject.Provides import com.twitter.cr_mixer.thriftscala.GetTweetsRecommendationsScribe import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finatra.kafka.producers.FinagleKafkaProducerBuilder import com.twitter.finatra.kafka.producers.KafkaProducerBase import com.twitter.finatra.kafka.producers.NullKafkaProducer import com.twitter.finatra.kafka.serde.ScalaSerdes import com.twitter.inject.TwitterModule import javax.inject.Singleton import org.apache.kafka.clients.CommonClientConfigs import org.apache.kafka.common.config.SaslConfigs import org.apache.kafka.common.config.SslConfigs import org.apache.kafka.common.record.CompressionType import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.serialization.Serdes object KafkaProducerModule extends TwitterModule { @Provides @Singleton def provideTweetRecsLoggerFactory( serviceIdentifier: ServiceIdentifier, ): KafkaProducerBase[String, GetTweetsRecommendationsScribe] = { KafkaProducerFactory.getKafkaProducer(serviceIdentifier.environment) } } object KafkaProducerFactory { private val jaasConfig = """com.sun.security.auth.module.Krb5LoginModule |required |principal="cr-mixer@TWITTER.BIZ" |debug=true |useKeyTab=true |storeKey=true |keyTab="/var/lib/tss/keys/fluffy/keytabs/client/cr-mixer.keytab" |doNotPrompt=true; """.stripMargin.replaceAll("\n", " ") private val trustStoreLocation = "/etc/tw_truststore/messaging/kafka/client.truststore.jks" def getKafkaProducer( environment: String ): KafkaProducerBase[String, GetTweetsRecommendationsScribe] = { if (environment == "prod") { FinagleKafkaProducerBuilder() .dest("/s/kafka/recommendations:kafka-tls") // kerberos params .withConfig(SaslConfigs.SASL_JAAS_CONFIG, jaasConfig) .withConfig( CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_SSL.toString) .withConfig(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, trustStoreLocation) .withConfig(SaslConfigs.SASL_MECHANISM, SaslConfigs.GSSAPI_MECHANISM) .withConfig(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "kafka") .withConfig(SaslConfigs.SASL_KERBEROS_SERVER_NAME, "kafka") // Kafka params .keySerializer(Serdes.String.serializer) .valueSerializer(ScalaSerdes.CompactThrift[GetTweetsRecommendationsScribe].serializer()) .clientId("cr-mixer") .enableIdempotence(true) .compressionType(CompressionType.LZ4) .build() } else { new NullKafkaProducer[String, GetTweetsRecommendationsScribe] } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/LoggerFactoryModule.scala ================================================ package com.twitter.cr_mixer.module.core import com.google.inject.Provides import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.scribe.ScribeCategories import com.twitter.cr_mixer.scribe.ScribeCategory import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.logging.BareFormatter import com.twitter.logging.Level import com.twitter.logging.Logger import com.twitter.logging.NullHandler import com.twitter.logging.QueueingHandler import com.twitter.logging.ScribeHandler import com.twitter.logging.{LoggerFactory => TwitterLoggerFactory} import javax.inject.Named import javax.inject.Singleton object LoggerFactoryModule extends TwitterModule { private val DefaultQueueSize = 10000 @Provides @Singleton @Named(ModuleNames.AbDeciderLogger) def provideAbDeciderLogger( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Logger = { buildLoggerFactory( ScribeCategories.AbDecider, serviceIdentifier.environment, statsReceiver.scope("ScribeLogger")) .apply() } @Provides @Singleton @Named(ModuleNames.TopLevelApiDdgMetricsLogger) def provideTopLevelApiDdgMetricsLogger( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Logger = { buildLoggerFactory( ScribeCategories.TopLevelApiDdgMetrics, serviceIdentifier.environment, statsReceiver.scope("ScribeLogger")) .apply() } @Provides @Singleton @Named(ModuleNames.TweetRecsLogger) def provideTweetRecsLogger( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Logger = { buildLoggerFactory( ScribeCategories.TweetsRecs, serviceIdentifier.environment, statsReceiver.scope("ScribeLogger")) .apply() } @Provides @Singleton @Named(ModuleNames.BlueVerifiedTweetRecsLogger) def provideVITTweetRecsLogger( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Logger = { buildLoggerFactory( ScribeCategories.VITTweetsRecs, serviceIdentifier.environment, statsReceiver.scope("ScribeLogger")) .apply() } @Provides @Singleton @Named(ModuleNames.RelatedTweetsLogger) def provideRelatedTweetsLogger( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Logger = { buildLoggerFactory( ScribeCategories.RelatedTweets, serviceIdentifier.environment, statsReceiver.scope("ScribeLogger")) .apply() } @Provides @Singleton @Named(ModuleNames.UtegTweetsLogger) def provideUtegTweetsLogger( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Logger = { buildLoggerFactory( ScribeCategories.UtegTweets, serviceIdentifier.environment, statsReceiver.scope("ScribeLogger")) .apply() } @Provides @Singleton @Named(ModuleNames.AdsRecommendationsLogger) def provideAdsRecommendationsLogger( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Logger = { buildLoggerFactory( ScribeCategories.AdsRecommendations, serviceIdentifier.environment, statsReceiver.scope("ScribeLogger")) .apply() } private def buildLoggerFactory( category: ScribeCategory, environment: String, statsReceiver: StatsReceiver ): TwitterLoggerFactory = { environment match { case "prod" => TwitterLoggerFactory( node = category.getProdLoggerFactoryNode, level = Some(Level.INFO), useParents = false, handlers = List( QueueingHandler( maxQueueSize = DefaultQueueSize, handler = ScribeHandler( category = category.scribeCategory, formatter = BareFormatter, statsReceiver = statsReceiver.scope(category.getProdLoggerFactoryNode) ) ) ) ) case _ => TwitterLoggerFactory( node = category.getStagingLoggerFactoryNode, level = Some(Level.DEBUG), useParents = false, handlers = List( { () => NullHandler } ) ) } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/MemoizingStatsReceiverModule.scala ================================================ package com.twitter.cr_mixer.module.core import com.twitter.finagle.stats.LoadedStatsReceiver import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.servo.util.MemoizingStatsReceiver object MemoizingStatsReceiverModule extends TwitterModule { override def configure(): Unit = { bind[StatsReceiver].toInstance(new MemoizingStatsReceiver(LoadedStatsReceiver)) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/TimeoutConfigModule.scala ================================================ package com.twitter.cr_mixer.module.core import com.twitter.inject.TwitterModule import com.google.inject.Provides import javax.inject.Singleton import com.twitter.util.Duration import com.twitter.app.Flag import com.twitter.cr_mixer.config.TimeoutConfig /** * All timeout settings in CrMixer. * Timeout numbers are defined in source/cr-mixer/server/config/deploy.aurora */ object TimeoutConfigModule extends TwitterModule { /** * Flag names for client timeout * These are used in modules extending ThriftMethodBuilderClientModule * which cannot accept injection of TimeoutConfig */ val EarlybirdClientTimeoutFlagName = "earlybird.client.timeout" val FrsClientTimeoutFlagName = "frsSignalFetch.client.timeout" val QigRankerClientTimeoutFlagName = "qigRanker.client.timeout" val TweetypieClientTimeoutFlagName = "tweetypie.client.timeout" val UserTweetGraphClientTimeoutFlagName = "userTweetGraph.client.timeout" val UserTweetGraphPlusClientTimeoutFlagName = "userTweetGraphPlus.client.timeout" val UserAdGraphClientTimeoutFlagName = "userAdGraph.client.timeout" val UserVideoGraphClientTimeoutFlagName = "userVideoGraph.client.timeout" val UtegClientTimeoutFlagName = "uteg.client.timeout" val NaviRequestTimeoutFlagName = "navi.client.request.timeout" /** * Flags for timeouts * These are defined and initialized only in this file */ // timeout for the service private val serviceTimeout: Flag[Duration] = flag("service.timeout", "service total timeout") // timeout for signal fetch private val signalFetchTimeout: Flag[Duration] = flag[Duration]("signalFetch.timeout", "signal fetch timeout") // timeout for similarity engine private val similarityEngineTimeout: Flag[Duration] = flag[Duration]("similarityEngine.timeout", "similarity engine timeout") private val annServiceClientTimeout: Flag[Duration] = flag[Duration]("annService.client.timeout", "annQueryService client timeout") // timeout for user affinities fetcher private val userStateUnderlyingStoreTimeout: Flag[Duration] = flag[Duration]("userStateUnderlyingStore.timeout", "user state underlying store timeout") private val userStateStoreTimeout: Flag[Duration] = flag[Duration]("userStateStore.timeout", "user state store timeout") private val utegSimilarityEngineTimeout: Flag[Duration] = flag[Duration]("uteg.similarityEngine.timeout", "uteg similarity engine timeout") private val earlybirdServerTimeout: Flag[Duration] = flag[Duration]("earlybird.server.timeout", "earlybird server timeout") private val earlybirdSimilarityEngineTimeout: Flag[Duration] = flag[Duration]("earlybird.similarityEngine.timeout", "Earlybird similarity engine timeout") private val frsBasedTweetEndpointTimeout: Flag[Duration] = flag[Duration]( "frsBasedTweet.endpoint.timeout", "frsBasedTweet endpoint timeout" ) private val topicTweetEndpointTimeout: Flag[Duration] = flag[Duration]( "topicTweet.endpoint.timeout", "topicTweet endpoint timeout" ) // timeout for Navi client private val naviRequestTimeout: Flag[Duration] = flag[Duration]( NaviRequestTimeoutFlagName, Duration.fromMilliseconds(2000), "Request timeout for a single RPC Call", ) @Provides @Singleton def provideTimeoutBudget(): TimeoutConfig = TimeoutConfig( serviceTimeout = serviceTimeout(), signalFetchTimeout = signalFetchTimeout(), similarityEngineTimeout = similarityEngineTimeout(), annServiceClientTimeout = annServiceClientTimeout(), utegSimilarityEngineTimeout = utegSimilarityEngineTimeout(), userStateUnderlyingStoreTimeout = userStateUnderlyingStoreTimeout(), userStateStoreTimeout = userStateStoreTimeout(), earlybirdServerTimeout = earlybirdServerTimeout(), earlybirdSimilarityEngineTimeout = earlybirdSimilarityEngineTimeout(), frsBasedTweetEndpointTimeout = frsBasedTweetEndpointTimeout(), topicTweetEndpointTimeout = topicTweetEndpointTimeout(), naviRequestTimeout = naviRequestTimeout() ) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/grpc_client/NaviGRPCClientModule.scala ================================================ package com.twitter.cr_mixer.module.grpc_client import com.google.inject.Provides import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.finagle.Http import com.twitter.finagle.grpc.FinagleChannelBuilder import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsStackClientSyntax import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.util.Duration import io.grpc.ManagedChannel import javax.inject.Named import javax.inject.Singleton object NaviGRPCClientModule extends TwitterModule { val maxRetryAttempts = 3 @Provides @Singleton @Named(ModuleNames.HomeNaviGRPCClient) def providesHomeNaviGRPCClient( serviceIdentifier: ServiceIdentifier, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, ): ManagedChannel = { val label = "navi-wals-recommended-tweets-home-client" val dest = "/s/ads-prediction/navi-wals-recommended-tweets-home" buildClient(serviceIdentifier, timeoutConfig, statsReceiver, dest, label) } @Provides @Singleton @Named(ModuleNames.AdsFavedNaviGRPCClient) def providesAdsFavedNaviGRPCClient( serviceIdentifier: ServiceIdentifier, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, ): ManagedChannel = { val label = "navi-wals-ads-faved-tweets" val dest = "/s/ads-prediction/navi-wals-ads-faved-tweets" buildClient(serviceIdentifier, timeoutConfig, statsReceiver, dest, label) } @Provides @Singleton @Named(ModuleNames.AdsMonetizableNaviGRPCClient) def providesAdsMonetizableNaviGRPCClient( serviceIdentifier: ServiceIdentifier, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, ): ManagedChannel = { val label = "navi-wals-ads-monetizable-tweets" val dest = "/s/ads-prediction/navi-wals-ads-monetizable-tweets" buildClient(serviceIdentifier, timeoutConfig, statsReceiver, dest, label) } private def buildClient( serviceIdentifier: ServiceIdentifier, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, dest: String, label: String ): ManagedChannel = { val stats = statsReceiver.scope("clnt").scope(label) val client = Http.client .withLabel(label) .withMutualTls(serviceIdentifier) .withRequestTimeout(timeoutConfig.naviRequestTimeout) .withTransport.connectTimeout(Duration.fromMilliseconds(10000)) .withSession.acquisitionTimeout(Duration.fromMilliseconds(20000)) .withStatsReceiver(stats) .withHttpStats FinagleChannelBuilder .forTarget(dest) .overrideAuthority("rustserving") .maxRetryAttempts(maxRetryAttempts) .enableRetryForStatus(io.grpc.Status.RESOURCE_EXHAUSTED) .enableRetryForStatus(io.grpc.Status.UNKNOWN) .enableUnsafeFullyBufferingMode() .httpClient(client) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/CertoTopicTweetSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TopicTweetWithScore import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.similarity_engine.CertoTopicTweetSimilarityEngine import com.twitter.cr_mixer.similarity_engine.CertoTopicTweetSimilarityEngine.Query import com.twitter.cr_mixer.similarity_engine.EngineQuery import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.thriftscala.TopicId import com.twitter.storehaus.ReadableStore import com.twitter.topic_recos.thriftscala.TweetWithScores import javax.inject.Named import javax.inject.Singleton object CertoTopicTweetSimilarityEngineModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.CertoTopicTweetSimilarityEngine) def providesCertoTopicTweetSimilarityEngine( @Named(ModuleNames.CertoStratoStoreName) certoStratoStore: ReadableStore[ TopicId, Seq[TweetWithScores] ], timeoutConfig: TimeoutConfig, decider: CrMixerDecider, statsReceiver: StatsReceiver ): StandardSimilarityEngine[ EngineQuery[Query], TopicTweetWithScore ] = { new StandardSimilarityEngine[EngineQuery[Query], TopicTweetWithScore]( implementingStore = CertoTopicTweetSimilarityEngine(certoStratoStore, statsReceiver), identifier = SimilarityEngineType.CertoTopicTweet, globalStats = statsReceiver, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.topicTweetEndpointTimeout, gatingConfig = GatingConfig( deciderConfig = Some(DeciderConfig(decider, DeciderConstants.enableTopicTweetTrafficDeciderKey)), enableFeatureSwitch = None ) ) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerBasedWalsSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.similarity_engine.ConsumerBasedWalsSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import io.grpc.ManagedChannel import javax.inject.Named object ConsumerBasedWalsSimilarityEngineModule extends TwitterModule { @Provides @Named(ModuleNames.ConsumerBasedWalsSimilarityEngine) def providesConsumerBasedWalsSimilarityEngine( timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, @Named(ModuleNames.HomeNaviGRPCClient) homeNaviGRPCClient: ManagedChannel, @Named(ModuleNames.AdsFavedNaviGRPCClient) adsFavedNaviGRPCClient: ManagedChannel, @Named(ModuleNames.AdsMonetizableNaviGRPCClient) adsMonetizableNaviGRPCClient: ManagedChannel, ): StandardSimilarityEngine[ ConsumerBasedWalsSimilarityEngine.Query, TweetWithScore ] = { val underlyingStore = new ConsumerBasedWalsSimilarityEngine( homeNaviGRPCClient, adsFavedNaviGRPCClient, adsMonetizableNaviGRPCClient, statsReceiver ) new StandardSimilarityEngine[ ConsumerBasedWalsSimilarityEngine.Query, TweetWithScore ]( implementingStore = underlyingStore, identifier = SimilarityEngineType.ConsumerBasedWalsANN, globalStats = statsReceiver, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = None, enableFeatureSwitch = None ) ) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTripSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.ModelConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TripTweetWithScore import com.twitter.cr_mixer.similarity_engine.ConsumerEmbeddingBasedTripSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.similarity_engine.TripEngineQuery import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.hermit.store.common.ObservedReadableStore import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.common.SimClustersEmbedding import com.twitter.simclusters_v2.common.UserId import com.twitter.storehaus.ReadableStore import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweet import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain import javax.inject.Named object ConsumerEmbeddingBasedTripSimilarityEngineModule extends TwitterModule { @Provides @Named(ModuleNames.ConsumerEmbeddingBasedTripSimilarityEngine) def providesConsumerEmbeddingBasedTripSimilarityEngineModule( @Named(ModuleNames.RmsUserLogFavInterestedInEmbeddingStore) userLogFavInterestedInEmbeddingStore: ReadableStore[UserId, SimClustersEmbedding], @Named(ModuleNames.RmsUserFollowInterestedInEmbeddingStore) userFollowInterestedInEmbeddingStore: ReadableStore[UserId, SimClustersEmbedding], @Named(ModuleNames.TripCandidateStore) tripCandidateStore: ReadableStore[TripDomain, Seq[TripTweet]], timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, ): StandardSimilarityEngine[TripEngineQuery, TripTweetWithScore] = { val underlyingStore = ObservedReadableStore( ConsumerEmbeddingBasedTripSimilarityEngine( embeddingStoreLookUpMap = Map( ModelConfig.ConsumerLogFavBasedInterestedInEmbedding -> userLogFavInterestedInEmbeddingStore, ModelConfig.ConsumerFollowBasedInterestedInEmbedding -> userFollowInterestedInEmbeddingStore, ), tripCandidateSource = tripCandidateStore, statsReceiver ))(statsReceiver.scope("TripSimilarityEngine")) new StandardSimilarityEngine[TripEngineQuery, TripTweetWithScore]( implementingStore = underlyingStore, identifier = SimilarityEngineType.ExploreTripOfflineSimClustersTweets, globalStats = statsReceiver, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = None, enableFeatureSwitch = None ) ) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTwHINSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.ann.common.thriftscala.AnnQueryService import com.twitter.cr_mixer.model.ModelConfig import com.twitter.cr_mixer.module.EmbeddingStoreModule import com.twitter.cr_mixer.module.thrift_client.AnnQueryServiceClientModule import com.twitter.cr_mixer.similarity_engine.HnswANNSimilarityEngine import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import javax.inject.Named import com.twitter.ml.api.{thriftscala => api} import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.thriftscala.SimilarityEngineType object ConsumerEmbeddingBasedTwHINSimilarityEngineModule extends TwitterModule { @Provides @Named(ModuleNames.ConsumerEmbeddingBasedTwHINANNSimilarityEngine) def providesConsumerEmbeddingBasedTwHINANNSimilarityEngine( // MH stores @Named(EmbeddingStoreModule.ConsumerBasedTwHINEmbeddingRegularUpdateMhStoreName) consumerBasedTwHINEmbeddingRegularUpdateMhStore: ReadableStore[InternalId, api.Embedding], @Named(EmbeddingStoreModule.DebuggerDemoUserEmbeddingMhStoreName) debuggerDemoUserEmbeddingMhStore: ReadableStore[InternalId, api.Embedding], @Named(AnnQueryServiceClientModule.TwHINRegularUpdateAnnServiceClientName) twHINRegularUpdateAnnService: AnnQueryService.MethodPerEndpoint, @Named(AnnQueryServiceClientModule.DebuggerDemoAnnServiceClientName) debuggerDemoAnnService: AnnQueryService.MethodPerEndpoint, // Other configs timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver ): HnswANNSimilarityEngine = { new HnswANNSimilarityEngine( embeddingStoreLookUpMap = Map( ModelConfig.ConsumerBasedTwHINRegularUpdateAll20221024 -> consumerBasedTwHINEmbeddingRegularUpdateMhStore, ModelConfig.DebuggerDemo -> debuggerDemoUserEmbeddingMhStore, ), annServiceLookUpMap = Map( ModelConfig.ConsumerBasedTwHINRegularUpdateAll20221024 -> twHINRegularUpdateAnnService, ModelConfig.DebuggerDemo -> debuggerDemoAnnService, ), globalStats = statsReceiver, identifier = SimilarityEngineType.ConsumerEmbeddingBasedTwHINANN, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = None, enableFeatureSwitch = None ) ) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTwoTowerSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module package similarity_engine import com.google.inject.Provides import com.twitter.ann.common.thriftscala.AnnQueryService import com.twitter.cr_mixer.model.ModelConfig import com.twitter.cr_mixer.module.EmbeddingStoreModule import com.twitter.cr_mixer.module.thrift_client.AnnQueryServiceClientModule import com.twitter.cr_mixer.similarity_engine.HnswANNSimilarityEngine import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import javax.inject.Named import com.twitter.ml.api.{thriftscala => api} import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.thriftscala.SimilarityEngineType object ConsumerEmbeddingBasedTwoTowerSimilarityEngineModule extends TwitterModule { @Provides @Named(ModuleNames.ConsumerEmbeddingBasedTwoTowerANNSimilarityEngine) def providesConsumerEmbeddingBasedTwoTowerANNSimilarityEngine( @Named(EmbeddingStoreModule.TwoTowerFavConsumerEmbeddingMhStoreName) twoTowerFavConsumerEmbeddingMhStore: ReadableStore[InternalId, api.Embedding], @Named(AnnQueryServiceClientModule.TwoTowerFavAnnServiceClientName) twoTowerFavAnnService: AnnQueryService.MethodPerEndpoint, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver ): HnswANNSimilarityEngine = { new HnswANNSimilarityEngine( embeddingStoreLookUpMap = Map( ModelConfig.TwoTowerFavALL20220808 -> twoTowerFavConsumerEmbeddingMhStore, ), annServiceLookUpMap = Map( ModelConfig.TwoTowerFavALL20220808 -> twoTowerFavAnnService, ), globalStats = statsReceiver, identifier = SimilarityEngineType.ConsumerEmbeddingBasedTwoTowerANN, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = None, enableFeatureSwitch = None ) ) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumersBasedUserAdGraphSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.similarity_engine.ConsumersBasedUserAdGraphSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.recos.user_ad_graph.thriftscala.ConsumersBasedRelatedAdRequest import com.twitter.recos.user_ad_graph.thriftscala.RelatedAdResponse import com.twitter.storehaus.ReadableStore import javax.inject.Named import javax.inject.Singleton object ConsumersBasedUserAdGraphSimilarityEngineModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.ConsumersBasedUserAdGraphSimilarityEngine) def providesConsumersBasedUserAdGraphSimilarityEngine( @Named(ModuleNames.ConsumerBasedUserAdGraphStore) consumersBasedUserAdGraphStore: ReadableStore[ ConsumersBasedRelatedAdRequest, RelatedAdResponse ], timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, decider: CrMixerDecider ): StandardSimilarityEngine[ ConsumersBasedUserAdGraphSimilarityEngine.Query, TweetWithScore ] = { new StandardSimilarityEngine[ ConsumersBasedUserAdGraphSimilarityEngine.Query, TweetWithScore ]( implementingStore = ConsumersBasedUserAdGraphSimilarityEngine(consumersBasedUserAdGraphStore, statsReceiver), identifier = SimilarityEngineType.ConsumersBasedUserTweetGraph, globalStats = statsReceiver, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = Some(DeciderConfig(decider, DeciderConstants.enableUserTweetGraphTrafficDeciderKey)), enableFeatureSwitch = None ) ), memCacheConfig = None ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumersBasedUserVideoGraphSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.similarity_engine.ConsumersBasedUserVideoGraphSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.recos.user_video_graph.thriftscala.ConsumersBasedRelatedTweetRequest import com.twitter.recos.user_video_graph.thriftscala.RelatedTweetResponse import com.twitter.storehaus.ReadableStore import javax.inject.Named import javax.inject.Singleton object ConsumersBasedUserVideoGraphSimilarityEngineModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.ConsumersBasedUserVideoGraphSimilarityEngine) def providesConsumersBasedUserVideoGraphSimilarityEngine( @Named(ModuleNames.ConsumerBasedUserVideoGraphStore) consumersBasedUserVideoGraphStore: ReadableStore[ ConsumersBasedRelatedTweetRequest, RelatedTweetResponse ], timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, decider: CrMixerDecider ): StandardSimilarityEngine[ ConsumersBasedUserVideoGraphSimilarityEngine.Query, TweetWithScore ] = { new StandardSimilarityEngine[ ConsumersBasedUserVideoGraphSimilarityEngine.Query, TweetWithScore ]( implementingStore = ConsumersBasedUserVideoGraphSimilarityEngine( consumersBasedUserVideoGraphStore, statsReceiver), identifier = SimilarityEngineType.ConsumersBasedUserVideoGraph, globalStats = statsReceiver, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = Some(DeciderConfig(decider, DeciderConstants.enableUserVideoGraphTrafficDeciderKey)), enableFeatureSwitch = None ) ), memCacheConfig = None ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/DiffusionBasedSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module package similarity_engine import com.google.inject.Provides import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.ModelConfig import com.twitter.simclusters_v2.thriftscala.TweetsWithScore import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.similarity_engine.DiffusionBasedSimilarityEngine import com.twitter.cr_mixer.similarity_engine.DiffusionBasedSimilarityEngine.Query import com.twitter.cr_mixer.similarity_engine.LookupSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.storehaus.ReadableStore import javax.inject.Named import javax.inject.Singleton object DiffusionBasedSimilarityEngineModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.DiffusionBasedSimilarityEngine) def providesDiffusionBasedSimilarityEngineModule( @Named(ModuleNames.RetweetBasedDiffusionRecsMhStore) retweetBasedDiffusionRecsMhStore: ReadableStore[Long, TweetsWithScore], timeoutConfig: TimeoutConfig, globalStats: StatsReceiver ): LookupSimilarityEngine[Query, TweetWithScore] = { val versionedStoreMap = Map( ModelConfig.RetweetBasedDiffusion -> DiffusionBasedSimilarityEngine( retweetBasedDiffusionRecsMhStore, globalStats), ) new LookupSimilarityEngine[Query, TweetWithScore]( versionedStoreMap = versionedStoreMap, identifier = SimilarityEngineType.DiffusionBasedTweet, globalStats = globalStats, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = None, enableFeatureSwitch = None ) ) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/EarlybirdSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.similarity_engine.EarlybirdModelBasedSimilarityEngine import com.twitter.cr_mixer.similarity_engine.EarlybirdRecencyBasedSimilarityEngine import com.twitter.cr_mixer.similarity_engine.EarlybirdSimilarityEngine import com.twitter.cr_mixer.similarity_engine.EarlybirdTensorflowBasedSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import javax.inject.Singleton object EarlybirdSimilarityEngineModule extends TwitterModule { @Provides @Singleton def providesRecencyBasedEarlybirdSimilarityEngine( earlybirdRecencyBasedSimilarityEngine: EarlybirdRecencyBasedSimilarityEngine, timeoutConfig: TimeoutConfig, decider: CrMixerDecider, statsReceiver: StatsReceiver ): EarlybirdSimilarityEngine[ EarlybirdRecencyBasedSimilarityEngine.EarlybirdRecencyBasedSearchQuery, EarlybirdRecencyBasedSimilarityEngine ] = { new EarlybirdSimilarityEngine[ EarlybirdRecencyBasedSimilarityEngine.EarlybirdRecencyBasedSearchQuery, EarlybirdRecencyBasedSimilarityEngine ]( implementingStore = earlybirdRecencyBasedSimilarityEngine, identifier = SimilarityEngineType.EarlybirdRecencyBasedSimilarityEngine, globalStats = statsReceiver.scope(SimilarityEngineType.EarlybirdRecencyBasedSimilarityEngine.name), engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.earlybirdSimilarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = Some( DeciderConfig( decider = decider, deciderString = DeciderConstants.enableEarlybirdTrafficDeciderKey )), enableFeatureSwitch = None ) ) ) } @Provides @Singleton def providesModelBasedEarlybirdSimilarityEngine( earlybirdModelBasedSimilarityEngine: EarlybirdModelBasedSimilarityEngine, timeoutConfig: TimeoutConfig, decider: CrMixerDecider, statsReceiver: StatsReceiver ): EarlybirdSimilarityEngine[ EarlybirdModelBasedSimilarityEngine.EarlybirdModelBasedSearchQuery, EarlybirdModelBasedSimilarityEngine ] = { new EarlybirdSimilarityEngine[ EarlybirdModelBasedSimilarityEngine.EarlybirdModelBasedSearchQuery, EarlybirdModelBasedSimilarityEngine ]( implementingStore = earlybirdModelBasedSimilarityEngine, identifier = SimilarityEngineType.EarlybirdModelBasedSimilarityEngine, globalStats = statsReceiver.scope(SimilarityEngineType.EarlybirdModelBasedSimilarityEngine.name), engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.earlybirdSimilarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = Some( DeciderConfig( decider = decider, deciderString = DeciderConstants.enableEarlybirdTrafficDeciderKey )), enableFeatureSwitch = None ) ) ) } @Provides @Singleton def providesTensorflowBasedEarlybirdSimilarityEngine( earlybirdTensorflowBasedSimilarityEngine: EarlybirdTensorflowBasedSimilarityEngine, timeoutConfig: TimeoutConfig, decider: CrMixerDecider, statsReceiver: StatsReceiver ): EarlybirdSimilarityEngine[ EarlybirdTensorflowBasedSimilarityEngine.EarlybirdTensorflowBasedSearchQuery, EarlybirdTensorflowBasedSimilarityEngine ] = { new EarlybirdSimilarityEngine[ EarlybirdTensorflowBasedSimilarityEngine.EarlybirdTensorflowBasedSearchQuery, EarlybirdTensorflowBasedSimilarityEngine ]( implementingStore = earlybirdTensorflowBasedSimilarityEngine, identifier = SimilarityEngineType.EarlybirdTensorflowBasedSimilarityEngine, globalStats = statsReceiver.scope(SimilarityEngineType.EarlybirdTensorflowBasedSimilarityEngine.name), engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.earlybirdSimilarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = Some( DeciderConfig( decider = decider, deciderString = DeciderConstants.enableEarlybirdTrafficDeciderKey )), enableFeatureSwitch = None ) ) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUnifiedSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithCandidateGenerationInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.similarity_engine.ProducerBasedUserTweetGraphSimilarityEngine import com.twitter.cr_mixer.similarity_engine.ProducerBasedUnifiedSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.cr_mixer.similarity_engine.SimClustersANNSimilarityEngine import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.storehaus.ReadableStore import javax.inject.Named import javax.inject.Singleton object ProducerBasedUnifiedSimilarityEngineModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.ProducerBasedUnifiedSimilarityEngine) def providesProducerBasedUnifiedSimilarityEngine( @Named(ModuleNames.ProducerBasedUserTweetGraphSimilarityEngine) producerBasedUserTweetGraphSimilarityEngine: StandardSimilarityEngine[ ProducerBasedUserTweetGraphSimilarityEngine.Query, TweetWithScore ], @Named(ModuleNames.SimClustersANNSimilarityEngine) simClustersANNSimilarityEngine: StandardSimilarityEngine[ SimClustersANNSimilarityEngine.Query, TweetWithScore ], timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, ): StandardSimilarityEngine[ ProducerBasedUnifiedSimilarityEngine.Query, TweetWithCandidateGenerationInfo ] = { val underlyingStore: ReadableStore[ProducerBasedUnifiedSimilarityEngine.Query, Seq[ TweetWithCandidateGenerationInfo ]] = ProducerBasedUnifiedSimilarityEngine( producerBasedUserTweetGraphSimilarityEngine, simClustersANNSimilarityEngine, statsReceiver ) new StandardSimilarityEngine[ ProducerBasedUnifiedSimilarityEngine.Query, TweetWithCandidateGenerationInfo ]( implementingStore = underlyingStore, identifier = SimilarityEngineType.ProducerBasedUnifiedSimilarityEngine, globalStats = statsReceiver, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = None, enableFeatureSwitch = None ) ) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUserAdGraphSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.similarity_engine.ProducerBasedUserAdGraphSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine._ import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.keyHasher import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.recos.user_ad_graph.thriftscala.UserAdGraph import javax.inject.Named import javax.inject.Singleton object ProducerBasedUserAdGraphSimilarityEngineModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.ProducerBasedUserAdGraphSimilarityEngine) def providesProducerBasedUserAdGraphSimilarityEngine( userAdGraphService: UserAdGraph.MethodPerEndpoint, @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, decider: CrMixerDecider ): StandardSimilarityEngine[ ProducerBasedUserAdGraphSimilarityEngine.Query, TweetWithScore ] = { new StandardSimilarityEngine[ ProducerBasedUserAdGraphSimilarityEngine.Query, TweetWithScore ]( implementingStore = ProducerBasedUserAdGraphSimilarityEngine(userAdGraphService, statsReceiver), identifier = SimilarityEngineType.ProducerBasedUserAdGraph, globalStats = statsReceiver, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = Some(DeciderConfig(decider, DeciderConstants.enableUserAdGraphTrafficDeciderKey)), enableFeatureSwitch = None ) ), memCacheConfig = Some( MemCacheConfig( cacheClient = crMixerUnifiedCacheClient, ttl = 10.minutes, keyToString = { k => //Example Query CRMixer:ProducerBasedUTG:1234567890ABCDEF f"ProducerBasedUTG:${keyHasher.hashKey(k.toString.getBytes)}%X" } )) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUserTweetGraphSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.similarity_engine.ProducerBasedUserTweetGraphSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimilarityEngine._ import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.keyHasher import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.recos.user_tweet_graph.thriftscala.UserTweetGraph import javax.inject.Named import javax.inject.Singleton object ProducerBasedUserTweetGraphSimilarityEngineModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.ProducerBasedUserTweetGraphSimilarityEngine) def providesProducerBasedUserTweetGraphSimilarityEngine( userTweetGraphService: UserTweetGraph.MethodPerEndpoint, @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, decider: CrMixerDecider ): StandardSimilarityEngine[ ProducerBasedUserTweetGraphSimilarityEngine.Query, TweetWithScore ] = { new StandardSimilarityEngine[ ProducerBasedUserTweetGraphSimilarityEngine.Query, TweetWithScore ]( implementingStore = ProducerBasedUserTweetGraphSimilarityEngine(userTweetGraphService, statsReceiver), identifier = SimilarityEngineType.ProducerBasedUserTweetGraph, globalStats = statsReceiver, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = Some(DeciderConfig(decider, DeciderConstants.enableUserTweetGraphTrafficDeciderKey)), enableFeatureSwitch = None ) ), memCacheConfig = Some( MemCacheConfig( cacheClient = crMixerUnifiedCacheClient, ttl = 10.minutes, keyToString = { k => //Example Query CRMixer:ProducerBasedUTG:1234567890ABCDEF f"ProducerBasedUTG:${keyHasher.hashKey(k.toString.getBytes)}%X" } )) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/SimClustersANNSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.similarity_engine.SimClustersANNSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimClustersANNSimilarityEngine.Query import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.finagle.stats.StatsReceiver import com.twitter.hashing.KeyHasher import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.hermit.store.common.ObservedReadableStore import com.twitter.inject.TwitterModule import com.twitter.relevance_platform.common.injection.LZ4Injection import com.twitter.relevance_platform.common.injection.SeqObjectInjection import com.twitter.simclusters_v2.candidate_source.SimClustersANNCandidateSource.CacheableShortTTLEmbeddingTypes import com.twitter.simclustersann.thriftscala.SimClustersANNService import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import javax.inject.Named import javax.inject.Singleton object SimClustersANNSimilarityEngineModule extends TwitterModule { private val keyHasher: KeyHasher = KeyHasher.FNV1A_64 @Provides @Singleton @Named(ModuleNames.SimClustersANNSimilarityEngine) def providesProdSimClustersANNSimilarityEngine( @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, simClustersANNServiceNameToClientMapper: Map[String, SimClustersANNService.MethodPerEndpoint], timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver ): StandardSimilarityEngine[Query, TweetWithScore] = { val underlyingStore = SimClustersANNSimilarityEngine(simClustersANNServiceNameToClientMapper, statsReceiver) val observedReadableStore = ObservedReadableStore(underlyingStore)(statsReceiver.scope("SimClustersANNServiceStore")) val memCachedStore: ReadableStore[Query, Seq[TweetWithScore]] = ObservedMemcachedReadableStore .fromCacheClient( backingStore = observedReadableStore, cacheClient = crMixerUnifiedCacheClient, ttl = 10.minutes )( valueInjection = LZ4Injection.compose(SeqObjectInjection[TweetWithScore]()), statsReceiver = statsReceiver.scope("simclusters_ann_store_memcache"), keyToString = { k => //Example Query CRMixer:SCANN:1:2:1234567890ABCDEF:1234567890ABCDEF f"CRMixer:SCANN:${k.simClustersANNQuery.sourceEmbeddingId.embeddingType.getValue()}%X" + f":${k.simClustersANNQuery.sourceEmbeddingId.modelVersion.getValue()}%X" + f":${keyHasher.hashKey(k.simClustersANNQuery.sourceEmbeddingId.internalId.toString.getBytes)}%X" + f":${keyHasher.hashKey(k.simClustersANNQuery.config.toString.getBytes)}%X" } ) // Only cache the candidates if it's not Consumer-source. For example, TweetSource, // ProducerSource, TopicSource val wrapperStats = statsReceiver.scope("SimClustersANNWrapperStore") val wrapperStore: ReadableStore[Query, Seq[TweetWithScore]] = buildWrapperStore(memCachedStore, observedReadableStore, wrapperStats) new StandardSimilarityEngine[ Query, TweetWithScore ]( implementingStore = wrapperStore, identifier = SimilarityEngineType.SimClustersANN, globalStats = statsReceiver, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = None, enableFeatureSwitch = None ) ) ) } def buildWrapperStore( memCachedStore: ReadableStore[Query, Seq[TweetWithScore]], underlyingStore: ReadableStore[Query, Seq[TweetWithScore]], wrapperStats: StatsReceiver ): ReadableStore[Query, Seq[TweetWithScore]] = { // Only cache the candidates if it's not Consumer-source. For example, TweetSource, // ProducerSource, TopicSource val wrapperStore: ReadableStore[Query, Seq[TweetWithScore]] = new ReadableStore[Query, Seq[TweetWithScore]] { override def multiGet[K1 <: Query]( queries: Set[K1] ): Map[K1, Future[Option[Seq[TweetWithScore]]]] = { val (cacheableQueries, nonCacheableQueries) = queries.partition { query => CacheableShortTTLEmbeddingTypes.contains( query.simClustersANNQuery.sourceEmbeddingId.embeddingType) } memCachedStore.multiGet(cacheableQueries) ++ underlyingStore.multiGet(nonCacheableQueries) } } wrapperStore } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/SkitTopicTweetSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TopicTweetWithScore import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.similarity_engine.EngineQuery import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.SkitHighPrecisionTopicTweetSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SkitTopicTweetSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SkitTopicTweetSimilarityEngine.Query import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.storehaus.ReadableStore import com.twitter.topic_recos.thriftscala.TopicTweet import com.twitter.topic_recos.thriftscala.TopicTweetPartitionFlatKey import javax.inject.Named import javax.inject.Singleton object SkitTopicTweetSimilarityEngineModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.SkitHighPrecisionTopicTweetSimilarityEngine) def providesSkitHighPrecisionTopicTweetSimilarityEngine( @Named(ModuleNames.SkitStratoStoreName) skitStratoStore: ReadableStore[ TopicTweetPartitionFlatKey, Seq[TopicTweet] ], timeoutConfig: TimeoutConfig, decider: CrMixerDecider, statsReceiver: StatsReceiver ): StandardSimilarityEngine[ EngineQuery[Query], TopicTweetWithScore ] = { new StandardSimilarityEngine[EngineQuery[Query], TopicTweetWithScore]( implementingStore = SkitHighPrecisionTopicTweetSimilarityEngine(skitStratoStore, statsReceiver), identifier = SimilarityEngineType.SkitHighPrecisionTopicTweet, globalStats = statsReceiver.scope(SimilarityEngineType.SkitHighPrecisionTopicTweet.name), engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.topicTweetEndpointTimeout, gatingConfig = GatingConfig( deciderConfig = Some(DeciderConfig(decider, DeciderConstants.enableTopicTweetTrafficDeciderKey)), enableFeatureSwitch = None ) ) ) } @Provides @Singleton @Named(ModuleNames.SkitTopicTweetSimilarityEngine) def providesSkitTfgTopicTweetSimilarityEngine( @Named(ModuleNames.SkitStratoStoreName) skitStratoStore: ReadableStore[ TopicTweetPartitionFlatKey, Seq[TopicTweet] ], timeoutConfig: TimeoutConfig, decider: CrMixerDecider, statsReceiver: StatsReceiver ): StandardSimilarityEngine[ EngineQuery[Query], TopicTweetWithScore ] = { new StandardSimilarityEngine[EngineQuery[Query], TopicTweetWithScore]( implementingStore = SkitTopicTweetSimilarityEngine(skitStratoStore, statsReceiver), identifier = SimilarityEngineType.SkitTfgTopicTweet, globalStats = statsReceiver.scope(SimilarityEngineType.SkitTfgTopicTweet.name), engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.topicTweetEndpointTimeout, gatingConfig = GatingConfig( deciderConfig = Some(DeciderConfig(decider, DeciderConstants.enableTopicTweetTrafficDeciderKey)), enableFeatureSwitch = None ) ) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedQigSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.similarity_engine.SimilarityEngine._ import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.keyHasher import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.similarity_engine.TweetBasedQigSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.qig_ranker.thriftscala.QigRanker import javax.inject.Named import javax.inject.Singleton object TweetBasedQigSimilarityEngineModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.TweetBasedQigSimilarityEngine) def providesTweetBasedQigSimilarTweetsCandidateSource( qigRanker: QigRanker.MethodPerEndpoint, @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, decider: CrMixerDecider ): StandardSimilarityEngine[ TweetBasedQigSimilarityEngine.Query, TweetWithScore ] = { new StandardSimilarityEngine[ TweetBasedQigSimilarityEngine.Query, TweetWithScore ]( implementingStore = TweetBasedQigSimilarityEngine(qigRanker, statsReceiver), identifier = SimilarityEngineType.Qig, globalStats = statsReceiver, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = Some(DeciderConfig(decider, DeciderConstants.enableQigSimilarTweetsTrafficDeciderKey)), enableFeatureSwitch = None ) ), memCacheConfig = Some( MemCacheConfig( cacheClient = crMixerUnifiedCacheClient, ttl = 10.minutes, keyToString = { k => f"TweetBasedQIGRanker:${keyHasher.hashKey(k.sourceId.toString.getBytes)}%X" } ) ) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedTwHINSimlarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.ann.common.thriftscala.AnnQueryService import com.twitter.cr_mixer.model.ModelConfig import com.twitter.cr_mixer.module.EmbeddingStoreModule import com.twitter.cr_mixer.module.thrift_client.AnnQueryServiceClientModule import com.twitter.cr_mixer.similarity_engine.HnswANNSimilarityEngine import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import javax.inject.Named import com.twitter.ml.api.{thriftscala => api} import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.similarity_engine.HnswANNEngineQuery import com.twitter.cr_mixer.similarity_engine.SimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.memcached.{Client => MemcachedClient} object TweetBasedTwHINSimlarityEngineModule extends TwitterModule { @Provides @Named(ModuleNames.TweetBasedTwHINANNSimilarityEngine) def providesTweetBasedTwHINANNSimilarityEngine( // MH stores @Named(EmbeddingStoreModule.TwHINEmbeddingRegularUpdateMhStoreName) twHINEmbeddingRegularUpdateMhStore: ReadableStore[InternalId, api.Embedding], @Named(EmbeddingStoreModule.DebuggerDemoTweetEmbeddingMhStoreName) debuggerDemoTweetEmbeddingMhStore: ReadableStore[InternalId, api.Embedding], // ANN clients @Named(AnnQueryServiceClientModule.TwHINRegularUpdateAnnServiceClientName) twHINRegularUpdateAnnService: AnnQueryService.MethodPerEndpoint, @Named(AnnQueryServiceClientModule.DebuggerDemoAnnServiceClientName) debuggerDemoAnnService: AnnQueryService.MethodPerEndpoint, // Other configs @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver ): HnswANNSimilarityEngine = { new HnswANNSimilarityEngine( embeddingStoreLookUpMap = Map( ModelConfig.TweetBasedTwHINRegularUpdateAll20221024 -> twHINEmbeddingRegularUpdateMhStore, ModelConfig.DebuggerDemo -> debuggerDemoTweetEmbeddingMhStore, ), annServiceLookUpMap = Map( ModelConfig.TweetBasedTwHINRegularUpdateAll20221024 -> twHINRegularUpdateAnnService, ModelConfig.DebuggerDemo -> debuggerDemoAnnService, ), globalStats = statsReceiver, identifier = SimilarityEngineType.TweetBasedTwHINANN, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = None, enableFeatureSwitch = None ) ), memCacheConfigOpt = Some( SimilarityEngine.MemCacheConfig[HnswANNEngineQuery]( cacheClient = crMixerUnifiedCacheClient, ttl = 30.minutes, keyToString = (query: HnswANNEngineQuery) => SimilarityEngine.keyHasher.hashKey(query.cacheKey.getBytes).toString )) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUnifiedSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithCandidateGenerationInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.similarity_engine.HnswANNSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimClustersANNSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.similarity_engine.TweetBasedQigSimilarityEngine import com.twitter.cr_mixer.similarity_engine.TweetBasedUnifiedSimilarityEngine import com.twitter.cr_mixer.similarity_engine.TweetBasedUserTweetGraphSimilarityEngine import com.twitter.cr_mixer.similarity_engine.TweetBasedUserVideoGraphSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.storehaus.ReadableStore import javax.inject.Named import javax.inject.Singleton object TweetBasedUnifiedSimilarityEngineModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.TweetBasedUnifiedSimilarityEngine) def providesTweetBasedUnifiedSimilarityEngine( @Named(ModuleNames.TweetBasedUserTweetGraphSimilarityEngine) tweetBasedUserTweetGraphSimilarityEngine: StandardSimilarityEngine[ TweetBasedUserTweetGraphSimilarityEngine.Query, TweetWithScore ], @Named(ModuleNames.TweetBasedUserVideoGraphSimilarityEngine) tweetBasedUserVideoGraphSimilarityEngine: StandardSimilarityEngine[ TweetBasedUserVideoGraphSimilarityEngine.Query, TweetWithScore ], @Named(ModuleNames.TweetBasedTwHINANNSimilarityEngine) tweetBasedTwHINANNSimilarityEngine: HnswANNSimilarityEngine, @Named(ModuleNames.TweetBasedQigSimilarityEngine) tweetBasedQigSimilarityEngine: StandardSimilarityEngine[ TweetBasedQigSimilarityEngine.Query, TweetWithScore ], @Named(ModuleNames.SimClustersANNSimilarityEngine) simClustersANNSimilarityEngine: StandardSimilarityEngine[ SimClustersANNSimilarityEngine.Query, TweetWithScore ], timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, ): StandardSimilarityEngine[ TweetBasedUnifiedSimilarityEngine.Query, TweetWithCandidateGenerationInfo ] = { val underlyingStore: ReadableStore[TweetBasedUnifiedSimilarityEngine.Query, Seq[ TweetWithCandidateGenerationInfo ]] = TweetBasedUnifiedSimilarityEngine( tweetBasedUserTweetGraphSimilarityEngine, tweetBasedUserVideoGraphSimilarityEngine, simClustersANNSimilarityEngine, tweetBasedQigSimilarityEngine, tweetBasedTwHINANNSimilarityEngine, statsReceiver ) new StandardSimilarityEngine[ TweetBasedUnifiedSimilarityEngine.Query, TweetWithCandidateGenerationInfo ]( implementingStore = underlyingStore, identifier = SimilarityEngineType.TweetBasedUnifiedSimilarityEngine, globalStats = statsReceiver, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = None, enableFeatureSwitch = None ) ) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserAdGraphSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.similarity_engine.TweetBasedUserAdGraphSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.finagle.stats.StatsReceiver import com.twitter.hashing.KeyHasher import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.inject.TwitterModule import com.twitter.recos.user_ad_graph.thriftscala.UserAdGraph import com.twitter.relevance_platform.common.injection.LZ4Injection import com.twitter.relevance_platform.common.injection.SeqObjectInjection import com.twitter.simclusters_v2.common.TweetId import com.twitter.storehaus.ReadableStore import com.twitter.twistly.thriftscala.TweetRecentEngagedUsers import javax.inject.Named import javax.inject.Singleton object TweetBasedUserAdGraphSimilarityEngineModule extends TwitterModule { private val keyHasher: KeyHasher = KeyHasher.FNV1A_64 @Provides @Singleton @Named(ModuleNames.TweetBasedUserAdGraphSimilarityEngine) def providesTweetBasedUserAdGraphSimilarityEngine( userAdGraphService: UserAdGraph.MethodPerEndpoint, tweetRecentEngagedUserStore: ReadableStore[TweetId, TweetRecentEngagedUsers], @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, decider: CrMixerDecider ): StandardSimilarityEngine[ TweetBasedUserAdGraphSimilarityEngine.Query, TweetWithScore ] = { val underlyingStore = TweetBasedUserAdGraphSimilarityEngine( userAdGraphService, tweetRecentEngagedUserStore, statsReceiver) val memCachedStore: ReadableStore[ TweetBasedUserAdGraphSimilarityEngine.Query, Seq[ TweetWithScore ] ] = ObservedMemcachedReadableStore .fromCacheClient( backingStore = underlyingStore, cacheClient = crMixerUnifiedCacheClient, ttl = 10.minutes )( valueInjection = LZ4Injection.compose(SeqObjectInjection[TweetWithScore]()), statsReceiver = statsReceiver.scope("tweet_based_user_ad_graph_store_memcache"), keyToString = { k => //Example Query CRMixer:TweetBasedUTG:1234567890ABCDEF f"CRMixer:TweetBasedUAG:${keyHasher.hashKey(k.toString.getBytes)}%X" } ) new StandardSimilarityEngine[ TweetBasedUserAdGraphSimilarityEngine.Query, TweetWithScore ]( implementingStore = memCachedStore, identifier = SimilarityEngineType.TweetBasedUserAdGraph, globalStats = statsReceiver, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = Some(DeciderConfig(decider, DeciderConstants.enableUserAdGraphTrafficDeciderKey)), enableFeatureSwitch = None ) ) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserTweetGraphSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module package similarity_engine import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.similarity_engine.TweetBasedUserTweetGraphSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.finagle.stats.StatsReceiver import com.twitter.hashing.KeyHasher import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.inject.TwitterModule import com.twitter.recos.user_tweet_graph.thriftscala.UserTweetGraph import com.twitter.relevance_platform.common.injection.LZ4Injection import com.twitter.relevance_platform.common.injection.SeqObjectInjection import com.twitter.simclusters_v2.common.TweetId import com.twitter.storehaus.ReadableStore import com.twitter.twistly.thriftscala.TweetRecentEngagedUsers import javax.inject.Named import javax.inject.Singleton object TweetBasedUserTweetGraphSimilarityEngineModule extends TwitterModule { private val keyHasher: KeyHasher = KeyHasher.FNV1A_64 @Provides @Singleton @Named(ModuleNames.TweetBasedUserTweetGraphSimilarityEngine) def providesTweetBasedUserTweetGraphSimilarityEngine( userTweetGraphService: UserTweetGraph.MethodPerEndpoint, tweetRecentEngagedUserStore: ReadableStore[TweetId, TweetRecentEngagedUsers], @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, decider: CrMixerDecider ): StandardSimilarityEngine[ TweetBasedUserTweetGraphSimilarityEngine.Query, TweetWithScore ] = { val underlyingStore = TweetBasedUserTweetGraphSimilarityEngine( userTweetGraphService, tweetRecentEngagedUserStore, statsReceiver) val memCachedStore: ReadableStore[ TweetBasedUserTweetGraphSimilarityEngine.Query, Seq[ TweetWithScore ] ] = ObservedMemcachedReadableStore .fromCacheClient( backingStore = underlyingStore, cacheClient = crMixerUnifiedCacheClient, ttl = 10.minutes )( valueInjection = LZ4Injection.compose(SeqObjectInjection[TweetWithScore]()), statsReceiver = statsReceiver.scope("tweet_based_user_tweet_graph_store_memcache"), keyToString = { k => //Example Query CRMixer:TweetBasedUTG:1234567890ABCDEF f"CRMixer:TweetBasedUTG:${keyHasher.hashKey(k.toString.getBytes)}%X" } ) new StandardSimilarityEngine[ TweetBasedUserTweetGraphSimilarityEngine.Query, TweetWithScore ]( implementingStore = memCachedStore, identifier = SimilarityEngineType.TweetBasedUserTweetGraph, globalStats = statsReceiver, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = Some(DeciderConfig(decider, DeciderConstants.enableUserTweetGraphTrafficDeciderKey)), enableFeatureSwitch = None ) ) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserVideoGraphSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.similarity_engine.TweetBasedUserVideoGraphSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.finagle.stats.StatsReceiver import com.twitter.hashing.KeyHasher import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.inject.TwitterModule import com.twitter.recos.user_video_graph.thriftscala.UserVideoGraph import com.twitter.relevance_platform.common.injection.LZ4Injection import com.twitter.relevance_platform.common.injection.SeqObjectInjection import com.twitter.simclusters_v2.common.TweetId import com.twitter.storehaus.ReadableStore import com.twitter.twistly.thriftscala.TweetRecentEngagedUsers import javax.inject.Named import javax.inject.Singleton object TweetBasedUserVideoGraphSimilarityEngineModule extends TwitterModule { private val keyHasher: KeyHasher = KeyHasher.FNV1A_64 @Provides @Singleton @Named(ModuleNames.TweetBasedUserVideoGraphSimilarityEngine) def providesTweetBasedUserVideoGraphSimilarityEngine( userVideoGraphService: UserVideoGraph.MethodPerEndpoint, tweetRecentEngagedUserStore: ReadableStore[TweetId, TweetRecentEngagedUsers], @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, decider: CrMixerDecider ): StandardSimilarityEngine[ TweetBasedUserVideoGraphSimilarityEngine.Query, TweetWithScore ] = { val underlyingStore = TweetBasedUserVideoGraphSimilarityEngine( userVideoGraphService, tweetRecentEngagedUserStore, statsReceiver) val memCachedStore: ReadableStore[ TweetBasedUserVideoGraphSimilarityEngine.Query, Seq[ TweetWithScore ] ] = ObservedMemcachedReadableStore .fromCacheClient( backingStore = underlyingStore, cacheClient = crMixerUnifiedCacheClient, ttl = 10.minutes )( valueInjection = LZ4Injection.compose(SeqObjectInjection[TweetWithScore]()), statsReceiver = statsReceiver.scope("tweet_based_user_video_graph_store_memcache"), keyToString = { k => //Example Query CRMixer:TweetBasedUVG:1234567890ABCDEF f"CRMixer:TweetBasedUVG:${keyHasher.hashKey(k.toString.getBytes)}%X" } ) new StandardSimilarityEngine[ TweetBasedUserVideoGraphSimilarityEngine.Query, TweetWithScore ]( implementingStore = memCachedStore, identifier = SimilarityEngineType.TweetBasedUserVideoGraph, globalStats = statsReceiver, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = Some(DeciderConfig(decider, DeciderConstants.enableUserVideoGraphTrafficDeciderKey)), enableFeatureSwitch = None ) ) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TwhinCollabFilterLookupSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module package similarity_engine import com.google.inject.Provides import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.simclusters_v2.common.TweetId import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.model.ModelConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.similarity_engine.LookupSimilarityEngine import com.twitter.cr_mixer.similarity_engine.TwhinCollabFilterSimilarityEngine.Query import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.TwhinCollabFilterSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.storehaus.ReadableStore import javax.inject.Named import javax.inject.Singleton /** * TwhinCandidatesLookupSimilarityEngineModule routes the request to the corresponding * twhin based candidate store which follow the same pattern as TwHIN Collaborative Filtering. */ object TwhinCollabFilterLookupSimilarityEngineModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.TwhinCollabFilterSimilarityEngine) def providesTwhinCollabFilterLookupSimilarityEngineModule( @Named(ModuleNames.TwhinCollabFilterStratoStoreForFollow) twhinCollabFilterStratoStoreForFollow: ReadableStore[Long, Seq[TweetId]], @Named(ModuleNames.TwhinCollabFilterStratoStoreForEngagement) twhinCollabFilterStratoStoreForEngagement: ReadableStore[Long, Seq[TweetId]], @Named(ModuleNames.TwhinMultiClusterStratoStoreForFollow) twhinMultiClusterStratoStoreForFollow: ReadableStore[Long, Seq[TweetId]], @Named(ModuleNames.TwhinMultiClusterStratoStoreForEngagement) twhinMultiClusterStratoStoreForEngagement: ReadableStore[Long, Seq[TweetId]], timeoutConfig: TimeoutConfig, globalStats: StatsReceiver ): LookupSimilarityEngine[Query, TweetWithScore] = { val versionedStoreMap = Map( ModelConfig.TwhinCollabFilterForFollow -> TwhinCollabFilterSimilarityEngine( twhinCollabFilterStratoStoreForFollow, globalStats), ModelConfig.TwhinCollabFilterForEngagement -> TwhinCollabFilterSimilarityEngine( twhinCollabFilterStratoStoreForEngagement, globalStats), ModelConfig.TwhinMultiClusterForFollow -> TwhinCollabFilterSimilarityEngine( twhinMultiClusterStratoStoreForFollow, globalStats), ModelConfig.TwhinMultiClusterForEngagement -> TwhinCollabFilterSimilarityEngine( twhinMultiClusterStratoStoreForEngagement, globalStats), ) new LookupSimilarityEngine[Query, TweetWithScore]( versionedStoreMap = versionedStoreMap, identifier = SimilarityEngineType.TwhinCollabFilter, globalStats = globalStats, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.similarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = None, enableFeatureSwitch = None ) ) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/UserTweetEntityGraphSimilarityEngineModule.scala ================================================ package com.twitter.cr_mixer.module.similarity_engine import com.google.inject.Provides import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithScoreAndSocialProof import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.similarity_engine.UserTweetEntityGraphSimilarityEngine import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph import javax.inject.Named import javax.inject.Singleton object UserTweetEntityGraphSimilarityEngineModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.UserTweetEntityGraphSimilarityEngine) def providesUserTweetEntityGraphSimilarityEngine( userTweetEntityGraphService: UserTweetEntityGraph.MethodPerEndpoint, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, decider: CrMixerDecider ): StandardSimilarityEngine[ UserTweetEntityGraphSimilarityEngine.Query, TweetWithScoreAndSocialProof ] = { new StandardSimilarityEngine[ UserTweetEntityGraphSimilarityEngine.Query, TweetWithScoreAndSocialProof ]( implementingStore = UserTweetEntityGraphSimilarityEngine(userTweetEntityGraphService, statsReceiver), identifier = SimilarityEngineType.Uteg, globalStats = statsReceiver, engineConfig = SimilarityEngineConfig( timeout = timeoutConfig.utegSimilarityEngineTimeout, gatingConfig = GatingConfig( deciderConfig = Some( DeciderConfig(decider, DeciderConstants.enableUserTweetEntityGraphTrafficDeciderKey)), enableFeatureSwitch = None ) ), // We cannot use the key to cache anything in UTEG because the key contains a long list of userIds memCacheConfig = None ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/AnnQueryServiceClientModule.scala ================================================ package com.twitter.cr_mixer.module.thrift_client import com.google.inject.Provides import com.twitter.ann.common.thriftscala.AnnQueryService import com.twitter.conversions.DurationOps._ import com.twitter.conversions.PercentOps._ import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.finagle.ThriftMux import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.mtls.client.MtlsStackClient._ import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.thrift.ClientId import com.twitter.inject.TwitterModule import javax.inject.Named import javax.inject.Singleton object AnnQueryServiceClientModule extends TwitterModule { final val DebuggerDemoAnnServiceClientName = "DebuggerDemoAnnServiceClient" @Provides @Singleton @Named(DebuggerDemoAnnServiceClientName) def debuggerDemoAnnServiceClient( serviceIdentifier: ServiceIdentifier, clientId: ClientId, statsReceiver: StatsReceiver, timeoutConfig: TimeoutConfig, ): AnnQueryService.MethodPerEndpoint = { // This ANN is built from the embeddings in src/scala/com/twitter/wtf/beam/bq_embedding_export/sql/MlfExperimentalTweetEmbeddingScalaDataset.sql // Change the above sql if you want to build the index from a diff embedding val dest = "/s/cassowary/mlf-experimental-ann-service" val label = "experimental-ann" buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) } final val TwHINUuaAnnServiceClientName = "TwHINUuaAnnServiceClient" @Provides @Singleton @Named(TwHINUuaAnnServiceClientName) def twhinUuaAnnServiceClient( serviceIdentifier: ServiceIdentifier, clientId: ClientId, statsReceiver: StatsReceiver, timeoutConfig: TimeoutConfig, ): AnnQueryService.MethodPerEndpoint = { val dest = "/s/cassowary/twhin-uua-ann-service" val label = "twhin_uua_ann" buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) } final val TwHINRegularUpdateAnnServiceClientName = "TwHINRegularUpdateAnnServiceClient" @Provides @Singleton @Named(TwHINRegularUpdateAnnServiceClientName) def twHINRegularUpdateAnnServiceClient( serviceIdentifier: ServiceIdentifier, clientId: ClientId, statsReceiver: StatsReceiver, timeoutConfig: TimeoutConfig, ): AnnQueryService.MethodPerEndpoint = { val dest = "/s/cassowary/twhin-regular-update-ann-service" val label = "twhin_regular_update" buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) } final val TwoTowerFavAnnServiceClientName = "TwoTowerFavAnnServiceClient" @Provides @Singleton @Named(TwoTowerFavAnnServiceClientName) def twoTowerFavAnnServiceClient( serviceIdentifier: ServiceIdentifier, clientId: ClientId, statsReceiver: StatsReceiver, timeoutConfig: TimeoutConfig, ): AnnQueryService.MethodPerEndpoint = { val dest = "/s/cassowary/tweet-rec-two-tower-fav-ann" val label = "tweet_rec_two_tower_fav_ann" buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) } private def buildClient( serviceIdentifier: ServiceIdentifier, clientId: ClientId, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, dest: String, label: String ): AnnQueryService.MethodPerEndpoint = { val thriftClient = ThriftMux.client .withMutualTls(serviceIdentifier) .withClientId(clientId) .withLabel(label) .withStatsReceiver(statsReceiver) .withTransport.connectTimeout(500.milliseconds) .withSession.acquisitionTimeout(500.milliseconds) .methodBuilder(dest) .withTimeoutPerRequest(timeoutConfig.annServiceClientTimeout) .withRetryDisabled .idempotent(5.percent) .servicePerEndpoint[AnnQueryService.ServicePerEndpoint] ThriftMux.Client.methodPerEndpoint(thriftClient) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/EarlybirdSearchClientModule.scala ================================================ package com.twitter.cr_mixer.module.thrift_client import com.twitter.app.Flag import com.twitter.finagle.ThriftMux import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.search.earlybird.thriftscala.EarlybirdService import com.twitter.inject.Injector import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.module.core.TimeoutConfigModule.EarlybirdClientTimeoutFlagName import com.twitter.finagle.service.RetryBudget import com.twitter.util.Duration import org.apache.thrift.protocol.TCompactProtocol object EarlybirdSearchClientModule extends ThriftMethodBuilderClientModule[ EarlybirdService.ServicePerEndpoint, EarlybirdService.MethodPerEndpoint ] with MtlsClient { override def label: String = "earlybird" override def dest: String = "/s/earlybird-root-superroot/root-superroot" private val requestTimeoutFlag: Flag[Duration] = flag[Duration](EarlybirdClientTimeoutFlagName, "Earlybird client timeout") override protected def requestTimeout: Duration = requestTimeoutFlag() override def retryBudget: RetryBudget = RetryBudget.Empty override def configureThriftMuxClient( injector: Injector, client: ThriftMux.Client ): ThriftMux.Client = { super .configureThriftMuxClient(injector, client) .withProtocolFactory(new TCompactProtocol.Factory()) .withSessionQualifier .successRateFailureAccrual(successRate = 0.9, window = 30.seconds) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/FrsClientModule.scala ================================================ package com.twitter.cr_mixer.module.thrift_client import com.twitter.app.Flag import com.twitter.finagle.ThriftMux import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.module.core.TimeoutConfigModule.FrsClientTimeoutFlagName import com.twitter.finagle.service.RetryBudget import com.twitter.finagle.stats.StatsReceiver import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService import com.twitter.inject.Injector import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.util.Duration object FrsClientModule extends ThriftMethodBuilderClientModule[ FollowRecommendationsThriftService.ServicePerEndpoint, FollowRecommendationsThriftService.MethodPerEndpoint ] with MtlsClient { override def label: String = "follow-recommendations-service" override def dest: String = "/s/follow-recommendations/follow-recos-service" private val frsSignalFetchTimeout: Flag[Duration] = flag[Duration](FrsClientTimeoutFlagName, "FRS signal fetch client timeout") override def requestTimeout: Duration = frsSignalFetchTimeout() override def retryBudget: RetryBudget = RetryBudget.Empty override def configureThriftMuxClient( injector: Injector, client: ThriftMux.Client ): ThriftMux.Client = { super .configureThriftMuxClient(injector, client) .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) .withSessionQualifier .successRateFailureAccrual(successRate = 0.9, window = 30.seconds) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/HydraPartitionClientModule.scala ================================================ package com.twitter.cr_mixer.module.thrift_client import com.twitter.conversions.DurationOps._ import com.twitter.finagle.thriftmux.MethodBuilder import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.inject.Injector import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.hydra.partition.{thriftscala => ht} object HydraPartitionClientModule extends ThriftMethodBuilderClientModule[ ht.HydraPartition.ServicePerEndpoint, ht.HydraPartition.MethodPerEndpoint ] with MtlsClient { override def label: String = "hydra-partition" override def dest: String = "/s/hydra/hydra-partition" override protected def configureMethodBuilder( injector: Injector, methodBuilder: MethodBuilder ): MethodBuilder = methodBuilder.withTimeoutTotal(500.milliseconds) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/HydraRootClientModule.scala ================================================ package com.twitter.cr_mixer.module.thrift_client import com.twitter.conversions.DurationOps._ import com.twitter.finagle.thriftmux.MethodBuilder import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.hydra.root.{thriftscala => ht} import com.twitter.inject.Injector import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule object HydraRootClientModule extends ThriftMethodBuilderClientModule[ ht.HydraRoot.ServicePerEndpoint, ht.HydraRoot.MethodPerEndpoint ] with MtlsClient { override def label: String = "hydra-root" override def dest: String = "/s/hydra/hydra-root" override protected def configureMethodBuilder( injector: Injector, methodBuilder: MethodBuilder ): MethodBuilder = methodBuilder.withTimeoutTotal(500.milliseconds) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/QigServiceClientModule.scala ================================================ package com.twitter.cr_mixer.module.thrift_client import com.twitter.app.Flag import com.twitter.cr_mixer.module.core.TimeoutConfigModule.QigRankerClientTimeoutFlagName import com.twitter.finagle.ThriftMux import com.twitter.finagle.mux.ClientDiscardedRequestException import com.twitter.finagle.service.ReqRep import com.twitter.finagle.service.ResponseClass import com.twitter.finagle.stats.StatsReceiver import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.inject.Injector import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.qig_ranker.thriftscala.QigRanker import com.twitter.util.Duration import com.twitter.util.Throw object QigServiceClientModule extends ThriftMethodBuilderClientModule[ QigRanker.ServicePerEndpoint, QigRanker.MethodPerEndpoint ] with MtlsClient { override val label: String = "qig-ranker" override val dest: String = "/s/qig-shared/qig-ranker" private val qigRankerClientTimeout: Flag[Duration] = flag[Duration](QigRankerClientTimeoutFlagName, "ranking timeout") override def requestTimeout: Duration = qigRankerClientTimeout() override def configureThriftMuxClient( injector: Injector, client: ThriftMux.Client ): ThriftMux.Client = super .configureThriftMuxClient(injector, client) .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) .withResponseClassifier { case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/SimClustersAnnServiceClientModule.scala ================================================ package com.twitter.cr_mixer.module.thrift_client import com.google.inject.Provides import com.twitter.conversions.PercentOps._ import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.finagle.ThriftMux import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.mtls.client.MtlsStackClient._ import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.thrift.ClientId import com.twitter.inject.TwitterModule import com.twitter.simclustersann.{thriftscala => t} import javax.inject.Named import javax.inject.Singleton object SimClustersAnnServiceClientModule extends TwitterModule { @Provides @Singleton @Named(ModuleNames.ProdSimClustersANNServiceClientName) def providesProdSimClustersANNServiceClient( serviceIdentifier: ServiceIdentifier, clientId: ClientId, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, ): t.SimClustersANNService.MethodPerEndpoint = { val label = "simclusters-ann-server" val dest = "/s/simclusters-ann/simclusters-ann" buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) } @Provides @Singleton @Named(ModuleNames.ExperimentalSimClustersANNServiceClientName) def providesExperimentalSimClustersANNServiceClient( serviceIdentifier: ServiceIdentifier, clientId: ClientId, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, ): t.SimClustersANNService.MethodPerEndpoint = { val label = "simclusters-ann-experimental-server" val dest = "/s/simclusters-ann/simclusters-ann-experimental" buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) } @Provides @Singleton @Named(ModuleNames.SimClustersANNServiceClientName1) def providesSimClustersANNServiceClient1( serviceIdentifier: ServiceIdentifier, clientId: ClientId, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, ): t.SimClustersANNService.MethodPerEndpoint = { val label = "simclusters-ann-server-1" val dest = "/s/simclusters-ann/simclusters-ann-1" buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) } @Provides @Singleton @Named(ModuleNames.SimClustersANNServiceClientName2) def providesSimClustersANNServiceClient2( serviceIdentifier: ServiceIdentifier, clientId: ClientId, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, ): t.SimClustersANNService.MethodPerEndpoint = { val label = "simclusters-ann-server-2" val dest = "/s/simclusters-ann/simclusters-ann-2" buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) } @Provides @Singleton @Named(ModuleNames.SimClustersANNServiceClientName3) def providesSimClustersANNServiceClient3( serviceIdentifier: ServiceIdentifier, clientId: ClientId, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, ): t.SimClustersANNService.MethodPerEndpoint = { val label = "simclusters-ann-server-3" val dest = "/s/simclusters-ann/simclusters-ann-3" buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) } @Provides @Singleton @Named(ModuleNames.SimClustersANNServiceClientName5) def providesSimClustersANNServiceClient5( serviceIdentifier: ServiceIdentifier, clientId: ClientId, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, ): t.SimClustersANNService.MethodPerEndpoint = { val label = "simclusters-ann-server-5" val dest = "/s/simclusters-ann/simclusters-ann-5" buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) } @Provides @Singleton @Named(ModuleNames.SimClustersANNServiceClientName4) def providesSimClustersANNServiceClient4( serviceIdentifier: ServiceIdentifier, clientId: ClientId, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, ): t.SimClustersANNService.MethodPerEndpoint = { val label = "simclusters-ann-server-4" val dest = "/s/simclusters-ann/simclusters-ann-4" buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) } private def buildClient( serviceIdentifier: ServiceIdentifier, clientId: ClientId, timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver, dest: String, label: String ): t.SimClustersANNService.MethodPerEndpoint = { val stats = statsReceiver.scope("clnt") val thriftClient = ThriftMux.client .withMutualTls(serviceIdentifier) .withClientId(clientId) .withLabel(label) .withStatsReceiver(stats) .methodBuilder(dest) .idempotent(5.percent) .withTimeoutPerRequest(timeoutConfig.annServiceClientTimeout) .withRetryDisabled .servicePerEndpoint[t.SimClustersANNService.ServicePerEndpoint] ThriftMux.Client.methodPerEndpoint(thriftClient) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/TweetyPieClientModule.scala ================================================ package com.twitter.cr_mixer.module.thrift_client import com.google.inject.Provides import com.twitter.app.Flag import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.cr_mixer.module.core.TimeoutConfigModule.TweetypieClientTimeoutFlagName import com.twitter.finagle.ThriftMux import com.twitter.finagle.mux.ClientDiscardedRequestException import com.twitter.finagle.service.ReqRep import com.twitter.finagle.service.ResponseClass import com.twitter.finagle.service.RetryBudget import com.twitter.finagle.stats.StatsReceiver import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.inject.Injector import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.stitch.tweetypie.{TweetyPie => STweetyPie} import com.twitter.tweetypie.thriftscala.TweetService import com.twitter.util.Duration import com.twitter.util.Throw import javax.inject.Singleton object TweetyPieClientModule extends ThriftMethodBuilderClientModule[ TweetService.ServicePerEndpoint, TweetService.MethodPerEndpoint ] with MtlsClient { override val label = "tweetypie" override val dest = "/s/tweetypie/tweetypie" private val tweetypieClientTimeout: Flag[Duration] = flag[Duration](TweetypieClientTimeoutFlagName, "tweetypie client timeout") override def requestTimeout: Duration = tweetypieClientTimeout() override def retryBudget: RetryBudget = RetryBudget.Empty // We bump the success rate from the default of 0.8 to 0.9 since we're dropping the // consecutive failures part of the default policy. override def configureThriftMuxClient( injector: Injector, client: ThriftMux.Client ): ThriftMux.Client = super .configureThriftMuxClient(injector, client) .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) .withSessionQualifier .successRateFailureAccrual(successRate = 0.9, window = 30.seconds) .withResponseClassifier { case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable } @Provides @Singleton def providesTweetyPie( tweetyPieService: TweetService.MethodPerEndpoint ): STweetyPie = { STweetyPie(tweetyPieService) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserAdGraphClientModule.scala ================================================ package com.twitter.cr_mixer.module.thrift_client import com.twitter.app.Flag import com.twitter.cr_mixer.module.core.TimeoutConfigModule.UserAdGraphClientTimeoutFlagName import com.twitter.finagle.ThriftMux import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax import com.twitter.finagle.mux.ClientDiscardedRequestException import com.twitter.finagle.service.ReqRep import com.twitter.finagle.service.ResponseClass import com.twitter.finagle.service.RetryBudget import com.twitter.finagle.stats.StatsReceiver import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.inject.Injector import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.recos.user_ad_graph.thriftscala.UserAdGraph import com.twitter.util.Duration import com.twitter.util.Throw object UserAdGraphClientModule extends ThriftMethodBuilderClientModule[ UserAdGraph.ServicePerEndpoint, UserAdGraph.MethodPerEndpoint ] with MtlsClient { override val label = "user-ad-graph" override val dest = "/s/user-tweet-graph/user-ad-graph" private val userAdGraphClientTimeout: Flag[Duration] = flag[Duration](UserAdGraphClientTimeoutFlagName, "userAdGraph client timeout") override def requestTimeout: Duration = userAdGraphClientTimeout() override def retryBudget: RetryBudget = RetryBudget.Empty override def configureThriftMuxClient( injector: Injector, client: ThriftMux.Client ): ThriftMux.Client = super .configureThriftMuxClient(injector, client) .withMutualTls(injector.instance[ServiceIdentifier]) .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) .withResponseClassifier { case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetEntityGraphClientModule.scala ================================================ package com.twitter.cr_mixer.module.thrift_client import com.twitter.app.Flag import com.twitter.cr_mixer.module.core.TimeoutConfigModule.UtegClientTimeoutFlagName import com.twitter.finagle.ThriftMux import com.twitter.finagle.mux.ClientDiscardedRequestException import com.twitter.finagle.service.ReqRep import com.twitter.finagle.service.ResponseClass import com.twitter.finagle.service.RetryBudget import com.twitter.finagle.stats.StatsReceiver import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.inject.Injector import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph import com.twitter.util.Duration import com.twitter.util.Throw object UserTweetEntityGraphClientModule extends ThriftMethodBuilderClientModule[ UserTweetEntityGraph.ServicePerEndpoint, UserTweetEntityGraph.MethodPerEndpoint ] with MtlsClient { override val label = "user-tweet-entity-graph" override val dest = "/s/cassowary/user_tweet_entity_graph" private val userTweetEntityGraphClientTimeout: Flag[Duration] = flag[Duration](UtegClientTimeoutFlagName, "user tweet entity graph client timeout") override def requestTimeout: Duration = userTweetEntityGraphClientTimeout() override def retryBudget: RetryBudget = RetryBudget.Empty override def configureThriftMuxClient( injector: Injector, client: ThriftMux.Client ): ThriftMux.Client = super .configureThriftMuxClient(injector, client) .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) .withResponseClassifier { case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetGraphClientModule.scala ================================================ package com.twitter.cr_mixer.module.thrift_client import com.twitter.app.Flag import com.twitter.finagle.ThriftMux import com.twitter.finagle.mux.ClientDiscardedRequestException import com.twitter.finagle.service.ReqRep import com.twitter.finagle.service.ResponseClass import com.twitter.finagle.stats.StatsReceiver import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.inject.Injector import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.recos.user_tweet_graph.thriftscala.UserTweetGraph import com.twitter.util.Duration import com.twitter.util.Throw import com.twitter.cr_mixer.module.core.TimeoutConfigModule.UserTweetGraphClientTimeoutFlagName import com.twitter.finagle.service.RetryBudget object UserTweetGraphClientModule extends ThriftMethodBuilderClientModule[ UserTweetGraph.ServicePerEndpoint, UserTweetGraph.MethodPerEndpoint ] with MtlsClient { override val label = "user-tweet-graph" override val dest = "/s/user-tweet-graph/user-tweet-graph" private val userTweetGraphClientTimeout: Flag[Duration] = flag[Duration](UserTweetGraphClientTimeoutFlagName, "userTweetGraph client timeout") override def requestTimeout: Duration = userTweetGraphClientTimeout() override def retryBudget: RetryBudget = RetryBudget.Empty override def configureThriftMuxClient( injector: Injector, client: ThriftMux.Client ): ThriftMux.Client = super .configureThriftMuxClient(injector, client) .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) .withResponseClassifier { case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetGraphPlusClientModule.scala ================================================ package com.twitter.cr_mixer.module.thrift_client import com.twitter.app.Flag import com.twitter.cr_mixer.module.core.TimeoutConfigModule.UserTweetGraphPlusClientTimeoutFlagName import com.twitter.finagle.ThriftMux import com.twitter.finagle.mux.ClientDiscardedRequestException import com.twitter.finagle.service.ReqRep import com.twitter.finagle.service.ResponseClass import com.twitter.finagle.service.RetryBudget import com.twitter.finagle.stats.StatsReceiver import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.inject.Injector import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.recos.user_tweet_graph_plus.thriftscala.UserTweetGraphPlus import com.twitter.util.Duration import com.twitter.util.Throw object UserTweetGraphPlusClientModule extends ThriftMethodBuilderClientModule[ UserTweetGraphPlus.ServicePerEndpoint, UserTweetGraphPlus.MethodPerEndpoint ] with MtlsClient { override val label = "user-tweet-graph-plus" override val dest = "/s/user-tweet-graph/user-tweet-graph-plus" private val userTweetGraphPlusClientTimeout: Flag[Duration] = flag[Duration]( UserTweetGraphPlusClientTimeoutFlagName, "userTweetGraphPlus client timeout" ) override def requestTimeout: Duration = userTweetGraphPlusClientTimeout() override def retryBudget: RetryBudget = RetryBudget.Empty override def configureThriftMuxClient( injector: Injector, client: ThriftMux.Client ): ThriftMux.Client = super .configureThriftMuxClient(injector, client) .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) .withResponseClassifier { case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserVideoGraphClientModule.scala ================================================ package com.twitter.cr_mixer.module.thrift_client import com.twitter.app.Flag import com.twitter.cr_mixer.module.core.TimeoutConfigModule.UserVideoGraphClientTimeoutFlagName import com.twitter.finagle.ThriftMux import com.twitter.finagle.mux.ClientDiscardedRequestException import com.twitter.finagle.service.ReqRep import com.twitter.finagle.service.ResponseClass import com.twitter.finagle.service.RetryBudget import com.twitter.finagle.stats.StatsReceiver import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.inject.Injector import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.recos.user_video_graph.thriftscala.UserVideoGraph import com.twitter.util.Duration import com.twitter.util.Throw object UserVideoGraphClientModule extends ThriftMethodBuilderClientModule[ UserVideoGraph.ServicePerEndpoint, UserVideoGraph.MethodPerEndpoint ] with MtlsClient { override val label = "user-video-graph" override val dest = "/s/user-tweet-graph/user-video-graph" private val userVideoGraphClientTimeout: Flag[Duration] = flag[Duration]( UserVideoGraphClientTimeoutFlagName, "userVideoGraph client timeout" ) override def requestTimeout: Duration = userVideoGraphClientTimeout() override def retryBudget: RetryBudget = RetryBudget.Empty override def configureThriftMuxClient( injector: Injector, client: ThriftMux.Client ): ThriftMux.Client = super .configureThriftMuxClient(injector, client) .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) .withResponseClassifier { case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/AdsParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object AdsParams { object AdsCandidateGenerationMaxCandidatesNumParam extends FSBoundedParam[Int]( name = "ads_candidate_generation_max_candidates_num", default = 400, min = 0, max = 2000 ) object EnableScoreBoost extends FSParam[Boolean]( name = "ads_candidate_generation_enable_score_boost", default = false ) object AdsCandidateGenerationScoreBoostFactor extends FSBoundedParam[Double]( name = "ads_candidate_generation_score_boost_factor", default = 10000.0, min = 1.0, max = 100000.0 ) object EnableScribe extends FSParam[Boolean]( name = "ads_candidate_generation_enable_scribe", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq( AdsCandidateGenerationMaxCandidatesNumParam, EnableScoreBoost, AdsCandidateGenerationScoreBoostFactor ) lazy val config: BaseConfig = { val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( AdsCandidateGenerationMaxCandidatesNumParam) val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableScoreBoost, EnableScribe ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(AdsCandidateGenerationScoreBoostFactor) BaseConfigBuilder() .set(intOverrides: _*) .set(booleanOverrides: _*) .set(doubleOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/javax/inject:javax.inject", "abdecider/src/main/scala", "configapi/configapi-abdecider", "configapi/configapi-core", "configapi/configapi-featureswitches:v2", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", "cr-mixer/thrift/src/main/thrift:thrift-scala", "decider/src/main/scala", "discovery-common/src/main/scala/com/twitter/discovery/common/configapi", "featureswitches/featureswitches-core", "featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder", "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", "scribelib/marshallers/src/main/scala/com/twitter/scribelib/marshallers", "src/scala/com/twitter/simclusters_v2/common", "src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", "user-signal-service/thrift/src/main/thrift:thrift-scala", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BlenderParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.logging.Logger import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object BlenderParams { object BlendingAlgorithmEnum extends Enumeration { val RoundRobin: Value = Value val SourceTypeBackFill: Value = Value val SourceSignalSorting: Value = Value } object ContentBasedSortingAlgorithmEnum extends Enumeration { val FavoriteCount: Value = Value val SourceSignalRecency: Value = Value val RandomSorting: Value = Value val SimilarityToSignalSorting: Value = Value val CandidateRecency: Value = Value } object BlendingAlgorithmParam extends FSEnumParam[BlendingAlgorithmEnum.type]( name = "blending_algorithm_id", default = BlendingAlgorithmEnum.RoundRobin, enum = BlendingAlgorithmEnum ) object RankingInterleaveWeightShrinkageParam extends FSBoundedParam[Double]( name = "blending_enable_ml_ranking_interleave_weights_shrinkage", default = 1.0, min = 0.0, max = 1.0 ) object RankingInterleaveMaxWeightAdjustments extends FSBoundedParam[Int]( name = "blending_interleave_max_weighted_adjustments", default = 3000, min = 0, max = 9999 ) object SignalTypeSortingAlgorithmParam extends FSEnumParam[ContentBasedSortingAlgorithmEnum.type]( name = "blending_algorithm_inner_signal_sorting_id", default = ContentBasedSortingAlgorithmEnum.SourceSignalRecency, enum = ContentBasedSortingAlgorithmEnum ) object ContentBlenderTypeSortingAlgorithmParam extends FSEnumParam[ContentBasedSortingAlgorithmEnum.type]( name = "blending_algorithm_content_blender_sorting_id", default = ContentBasedSortingAlgorithmEnum.FavoriteCount, enum = ContentBasedSortingAlgorithmEnum ) //UserAffinities Algo Param: whether to distributed the source type weights object EnableDistributedSourceTypeWeightsParam extends FSParam[Boolean]( name = "blending_algorithm_enable_distributed_source_type_weights", default = false ) object BlendGroupingMethodEnum extends Enumeration { val SourceKeyDefault: Value = Value("SourceKey") val SourceTypeSimilarityEngine: Value = Value("SourceTypeSimilarityEngine") val AuthorId: Value = Value("AuthorId") } object BlendGroupingMethodParam extends FSEnumParam[BlendGroupingMethodEnum.type]( name = "blending_grouping_method_id", default = BlendGroupingMethodEnum.SourceKeyDefault, enum = BlendGroupingMethodEnum ) object RecencyBasedRandomSamplingHalfLifeInDays extends FSBoundedParam[Int]( name = "blending_interleave_random_sampling_recency_based_half_life_in_days", default = 7, min = 1, max = 28 ) object RecencyBasedRandomSamplingDefaultWeight extends FSBoundedParam[Double]( name = "blending_interleave_random_sampling_recency_based_default_weight", default = 1.0, min = 0.1, max = 2.0 ) object SourceTypeBackFillEnableVideoBackFill extends FSParam[Boolean]( name = "blending_enable_video_backfill", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq( BlendingAlgorithmParam, RankingInterleaveWeightShrinkageParam, RankingInterleaveMaxWeightAdjustments, EnableDistributedSourceTypeWeightsParam, BlendGroupingMethodParam, RecencyBasedRandomSamplingHalfLifeInDays, RecencyBasedRandomSamplingDefaultWeight, SourceTypeBackFillEnableVideoBackFill, SignalTypeSortingAlgorithmParam, ContentBlenderTypeSortingAlgorithmParam, ) lazy val config: BaseConfig = { val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( NullStatsReceiver, Logger(getClass), BlendingAlgorithmParam, BlendGroupingMethodParam, SignalTypeSortingAlgorithmParam, ContentBlenderTypeSortingAlgorithmParam ) val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableDistributedSourceTypeWeightsParam, SourceTypeBackFillEnableVideoBackFill ) val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( RankingInterleaveMaxWeightAdjustments, RecencyBasedRandomSamplingHalfLifeInDays ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( RankingInterleaveWeightShrinkageParam, RecencyBasedRandomSamplingDefaultWeight ) BaseConfigBuilder() .set(enumOverrides: _*) .set(booleanOverrides: _*) .set(intOverrides: _*) .set(doubleOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BypassInterleaveAndRankParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object BypassInterleaveAndRankParams { object EnableTwhinCollabFilterBypassParam extends FSParam[Boolean]( name = "bypass_interleave_and_rank_twhin_collab_filter", default = false ) object EnableTwoTowerBypassParam extends FSParam[Boolean]( name = "bypass_interleave_and_rank_two_tower", default = false ) object EnableConsumerBasedTwhinBypassParam extends FSParam[Boolean]( name = "bypass_interleave_and_rank_consumer_based_twhin", default = false ) object EnableConsumerBasedWalsBypassParam extends FSParam[Boolean]( name = "bypass_interleave_and_rank_consumer_based_wals", default = false ) object TwhinCollabFilterBypassPercentageParam extends FSBoundedParam[Double]( name = "bypass_interleave_and_rank_twhin_collab_filter_percentage", default = 0.0, min = 0.0, max = 1.0 ) object TwoTowerBypassPercentageParam extends FSBoundedParam[Double]( name = "bypass_interleave_and_rank_two_tower_percentage", default = 0.0, min = 0.0, max = 1.0 ) object ConsumerBasedTwhinBypassPercentageParam extends FSBoundedParam[Double]( name = "bypass_interleave_and_rank_consumer_based_twhin_percentage", default = 0.0, min = 0.0, max = 1.0 ) object ConsumerBasedWalsBypassPercentageParam extends FSBoundedParam[Double]( name = "bypass_interleave_and_rank_consumer_based_wals_percentage", default = 0.0, min = 0.0, max = 1.0 ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableTwhinCollabFilterBypassParam, EnableTwoTowerBypassParam, EnableConsumerBasedTwhinBypassParam, EnableConsumerBasedWalsBypassParam, TwhinCollabFilterBypassPercentageParam, TwoTowerBypassPercentageParam, ConsumerBasedTwhinBypassPercentageParam, ConsumerBasedWalsBypassPercentageParam, ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableTwhinCollabFilterBypassParam, EnableTwoTowerBypassParam, EnableConsumerBasedTwhinBypassParam, EnableConsumerBasedWalsBypassParam, ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( TwhinCollabFilterBypassPercentageParam, TwoTowerBypassPercentageParam, ConsumerBasedTwhinBypassPercentageParam, ConsumerBasedWalsBypassPercentageParam, ) BaseConfigBuilder() .set(booleanOverrides: _*) .set(doubleOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerBasedWalsParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.timelines.configapi.Param import com.twitter.util.Duration object ConsumerBasedWalsParams { object EnableSourceParam extends FSParam[Boolean]( name = "consumer_based_wals_enable_source", default = false ) object ModelNameParam extends FSParam[String]( name = "consumer_based_wals_model_name", default = "model_0" ) object WilyNsNameParam extends FSParam[String]( name = "consumer_based_wals_wily_ns_name", default = "" ) object ModelInputNameParam extends FSParam[String]( name = "consumer_based_wals_model_input_name", default = "examples" ) object ModelOutputNameParam extends FSParam[String]( name = "consumer_based_wals_model_output_name", default = "all_tweet_ids" ) object ModelSignatureNameParam extends FSParam[String]( name = "consumer_based_wals_model_signature_name", default = "serving_default" ) object MaxTweetSignalAgeHoursParam extends FSBoundedParam[Duration]( name = "consumer_based_wals_max_tweet_signal_age_hours", default = 72.hours, min = 1.hours, max = 720.hours ) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromHours } val AllParams: Seq[Param[_] with FSName] = Seq( EnableSourceParam, ModelNameParam, ModelInputNameParam, ModelOutputNameParam, ModelSignatureNameParam, MaxTweetSignalAgeHoursParam, WilyNsNameParam, ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam, ) val stringOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides( ModelNameParam, ModelInputNameParam, ModelOutputNameParam, ModelSignatureNameParam, WilyNsNameParam ) val boundedDurationFSOverrides = FeatureSwitchOverrideUtil.getBoundedDurationFSOverrides(MaxTweetSignalAgeHoursParam) BaseConfigBuilder() .set(booleanOverrides: _*) .set(stringOverrides: _*) .set(boundedDurationFSOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedCandidateGenerationParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object ConsumerEmbeddingBasedCandidateGenerationParams { object EnableTwHINParam extends FSParam[Boolean]( name = "consumer_embedding_based_candidate_generation_enable_twhin", default = false ) object EnableTwoTowerParam extends FSParam[Boolean]( name = "consumer_embedding_based_candidate_generation_enable_two_tower", default = false ) object EnableLogFavBasedSimClustersTripParam extends FSParam[Boolean]( name = "consumer_embedding_based_candidate_generation_enable_logfav_based_simclusters_trip", default = false ) object EnableFollowBasedSimClustersTripParam extends FSParam[Boolean]( name = "consumer_embedding_based_candidate_generation_enable_follow_based_simclusters_trip", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableTwHINParam, EnableTwoTowerParam, EnableFollowBasedSimClustersTripParam, EnableLogFavBasedSimClustersTripParam ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableTwHINParam, EnableTwoTowerParam, EnableFollowBasedSimClustersTripParam, EnableLogFavBasedSimClustersTripParam ) BaseConfigBuilder() .set(booleanOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTripParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object ConsumerEmbeddingBasedTripParams { object SourceIdParam extends FSParam[String]( name = "consumer_embedding_based_trip_source_id", default = "EXPLR_TOPK_VID_48H_V3") object MaxNumCandidatesParam extends FSBoundedParam[Int]( name = "consumer_embedding_based_trip_max_num_candidates", default = 80, min = 0, max = 200 ) val AllParams: Seq[Param[_] with FSName] = Seq( SourceIdParam, MaxNumCandidatesParam ) lazy val config: BaseConfig = { val stringFSOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides( SourceIdParam ) val intFSOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( MaxNumCandidatesParam ) BaseConfigBuilder() .set(stringFSOverrides: _*) .set(intFSOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTwHINParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.cr_mixer.model.ModelConfig import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.Param import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil object ConsumerEmbeddingBasedTwHINParams { object ModelIdParam extends FSParam[String]( name = "consumer_embedding_based_twhin_model_id", default = ModelConfig.ConsumerBasedTwHINRegularUpdateAll20221024, ) // Note: this default value does not match with ModelIds yet. This FS is a placeholder val AllParams: Seq[Param[_] with FSName] = Seq( ModelIdParam ) lazy val config: BaseConfig = { val stringFSOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides( ModelIdParam ) BaseConfigBuilder() .set(stringFSOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTwoTowerParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.cr_mixer.model.ModelConfig.TwoTowerFavALL20220808 import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object ConsumerEmbeddingBasedTwoTowerParams { object ModelIdParam extends FSParam[String]( name = "consumer_embedding_based_two_tower_model_id", default = TwoTowerFavALL20220808, ) // Note: this default value does not match with ModelIds yet. This FS is a placeholder val AllParams: Seq[Param[_] with FSName] = Seq( ModelIdParam ) lazy val config: BaseConfig = { val stringFSOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides( ModelIdParam ) BaseConfigBuilder() .set(stringFSOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserAdGraphParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object ConsumersBasedUserAdGraphParams { object EnableSourceParam extends FSParam[Boolean]( name = "consumers_based_user_ad_graph_enable_source", default = false ) // UTG-Lookalike object MinCoOccurrenceParam extends FSBoundedParam[Int]( name = "consumers_based_user_ad_graph_min_co_occurrence", default = 2, min = 0, max = 500 ) object MinScoreParam extends FSBoundedParam[Double]( name = "consumers_based_user_ad_graph_min_score", default = 0.0, min = 0.0, max = 10.0 ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableSourceParam, MinCoOccurrenceParam, MinScoreParam ) lazy val config: BaseConfig = { val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides(MinCoOccurrenceParam) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(MinScoreParam) val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides(EnableSourceParam) BaseConfigBuilder() .set(intOverrides: _*) .set(booleanOverrides: _*) .set(doubleOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserTweetGraphParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param /** * ConsumersBasedUserTweetGraph Params, there are multiple ways (e.g. FRS, RealGraphOon) to generate consumersSeedSet for ConsumersBasedUserTweetGraph * for now we allow flexibility in tuning UTG params for different consumersSeedSet generation algo by giving the param name {consumerSeedSetAlgo}{ParamName} */ object ConsumersBasedUserTweetGraphParams { object EnableSourceParam extends FSParam[Boolean]( name = "consumers_based_user_tweet_graph_enable_source", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableSourceParam, ) lazy val config: BaseConfig = { val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides() val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides() val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam ) BaseConfigBuilder() .set(intOverrides: _*) .set(booleanOverrides: _*) .set(doubleOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserVideoGraphParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param /** * ConsumersBasedUserVideoGraph Params: there are multiple ways (e.g. FRS, RealGraphIn) to generate consumersSeedSet for ConsumersBasedUserTweetGraph * for now we allow flexibility in tuning UVG params for different consumersSeedSet generation algo by giving the param name {consumerSeedSetAlgo}{ParamName} */ object ConsumersBasedUserVideoGraphParams { object EnableSourceParam extends FSParam[Boolean]( name = "consumers_based_user_video_graph_enable_source", default = false ) // UTG-RealGraphIN object RealGraphInMinCoOccurrenceParam extends FSBoundedParam[Int]( name = "consumers_based_user_video_graph_real_graph_in_min_co_occurrence", default = 3, min = 0, max = 500 ) object RealGraphInMinScoreParam extends FSBoundedParam[Double]( name = "consumers_based_user_video_graph_real_graph_in_min_score", default = 2.0, min = 0.0, max = 10.0 ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableSourceParam, RealGraphInMinCoOccurrenceParam, RealGraphInMinScoreParam ) lazy val config: BaseConfig = { val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides(RealGraphInMinCoOccurrenceParam) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(RealGraphInMinScoreParam) val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam ) BaseConfigBuilder() .set(intOverrides: _*) .set(booleanOverrides: _*) .set(doubleOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CrMixerParamConfig.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.CompositeConfig import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.Param object CrMixerParamConfig { lazy val config: CompositeConfig = new CompositeConfig( configs = Seq( AdsParams.config, BlenderParams.config, BypassInterleaveAndRankParams.config, RankerParams.config, ConsumerBasedWalsParams.config, ConsumerEmbeddingBasedCandidateGenerationParams.config, ConsumerEmbeddingBasedTripParams.config, ConsumerEmbeddingBasedTwHINParams.config, ConsumerEmbeddingBasedTwoTowerParams.config, ConsumersBasedUserAdGraphParams.config, ConsumersBasedUserTweetGraphParams.config, ConsumersBasedUserVideoGraphParams.config, CustomizedRetrievalBasedCandidateGenerationParams.config, CustomizedRetrievalBasedOfflineInterestedInParams.config, CustomizedRetrievalBasedFTROfflineInterestedInParams.config, CustomizedRetrievalBasedTwhinParams.config, EarlybirdFrsBasedCandidateGenerationParams.config, FrsParams.config, GlobalParams.config, InterestedInParams.config, ProducerBasedCandidateGenerationParams.config, ProducerBasedUserAdGraphParams.config, ProducerBasedUserTweetGraphParams.config, RecentFollowsParams.config, RecentNegativeSignalParams.config, RecentNotificationsParams.config, RecentOriginalTweetsParams.config, RecentReplyTweetsParams.config, RecentRetweetsParams.config, RecentTweetFavoritesParams.config, RelatedTweetGlobalParams.config, RelatedVideoTweetGlobalParams.config, RelatedTweetProducerBasedParams.config, RelatedTweetTweetBasedParams.config, RelatedVideoTweetTweetBasedParams.config, RealGraphInParams.config, RealGraphOonParams.config, RepeatedProfileVisitsParams.config, SimClustersANNParams.config, TopicTweetParams.config, TweetBasedCandidateGenerationParams.config, TweetBasedUserAdGraphParams.config, TweetBasedUserTweetGraphParams.config, TweetBasedUserVideoGraphParams.config, TweetSharesParams.config, TweetBasedTwHINParams.config, RealGraphOonParams.config, GoodTweetClickParams.config, GoodProfileClickParams.config, UtegTweetGlobalParams.config, VideoTweetFilterParams.config, VideoViewTweetsParams.config, UnifiedUSSSignalParams.config, ), simpleName = "CrMixerConfig" ) val allParams: Seq[Param[_] with FSName] = { AdsParams.AllParams ++ BlenderParams.AllParams ++ BypassInterleaveAndRankParams.AllParams ++ RankerParams.AllParams ++ ConsumerBasedWalsParams.AllParams ++ ConsumerEmbeddingBasedCandidateGenerationParams.AllParams ++ ConsumerEmbeddingBasedTripParams.AllParams ++ ConsumerEmbeddingBasedTwHINParams.AllParams ++ ConsumerEmbeddingBasedTwoTowerParams.AllParams ++ ConsumersBasedUserAdGraphParams.AllParams ++ ConsumersBasedUserTweetGraphParams.AllParams ++ ConsumersBasedUserVideoGraphParams.AllParams ++ CustomizedRetrievalBasedCandidateGenerationParams.AllParams ++ CustomizedRetrievalBasedOfflineInterestedInParams.AllParams ++ CustomizedRetrievalBasedFTROfflineInterestedInParams.AllParams ++ CustomizedRetrievalBasedTwhinParams.AllParams ++ EarlybirdFrsBasedCandidateGenerationParams.AllParams ++ FrsParams.AllParams ++ GlobalParams.AllParams ++ InterestedInParams.AllParams ++ ProducerBasedCandidateGenerationParams.AllParams ++ ProducerBasedUserAdGraphParams.AllParams ++ ProducerBasedUserTweetGraphParams.AllParams ++ RecentFollowsParams.AllParams ++ RecentNegativeSignalParams.AllParams ++ RecentNotificationsParams.AllParams ++ RecentOriginalTweetsParams.AllParams ++ RecentReplyTweetsParams.AllParams ++ RecentRetweetsParams.AllParams ++ RecentTweetFavoritesParams.AllParams ++ RelatedTweetGlobalParams.AllParams ++ RelatedVideoTweetGlobalParams.AllParams ++ RelatedTweetProducerBasedParams.AllParams ++ RelatedTweetTweetBasedParams.AllParams ++ RelatedVideoTweetTweetBasedParams.AllParams ++ RepeatedProfileVisitsParams.AllParams ++ SimClustersANNParams.AllParams ++ TopicTweetParams.AllParams ++ TweetBasedCandidateGenerationParams.AllParams ++ TweetBasedUserAdGraphParams.AllParams ++ TweetBasedUserTweetGraphParams.AllParams ++ TweetBasedUserVideoGraphParams.AllParams ++ TweetSharesParams.AllParams ++ TweetBasedTwHINParams.AllParams ++ RealGraphOonParams.AllParams ++ RealGraphInParams.AllParams ++ GoodTweetClickParams.AllParams ++ GoodProfileClickParams.AllParams ++ UtegTweetGlobalParams.AllParams ++ VideoTweetFilterParams.AllParams ++ VideoViewTweetsParams.AllParams ++ UnifiedUSSSignalParams.AllParams } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedCandidateGenerationParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.cr_mixer.model.ModelConfig import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object CustomizedRetrievalBasedCandidateGenerationParams { // Offline SimClusters InterestedIn params object EnableOfflineInterestedInParam extends FSParam[Boolean]( name = "customized_retrieval_based_candidate_generation_enable_offline_interestedin", default = false ) // Offline SimClusters FTR-based InterestedIn object EnableOfflineFTRInterestedInParam extends FSParam[Boolean]( name = "customized_retrieval_based_candidate_generation_enable_ftr_offline_interestedin", default = false ) // TwHin Collab Filter Cluster params object EnableTwhinCollabFilterClusterParam extends FSParam[Boolean]( name = "customized_retrieval_based_candidate_generation_enable_twhin_collab_filter_cluster", default = false ) // TwHin Multi Cluster params object EnableTwhinMultiClusterParam extends FSParam[Boolean]( name = "customized_retrieval_based_candidate_generation_enable_twhin_multi_cluster", default = false ) object EnableRetweetBasedDiffusionParam extends FSParam[Boolean]( name = "customized_retrieval_based_candidate_generation_enable_retweet_based_diffusion", default = false ) object CustomizedRetrievalBasedRetweetDiffusionSource extends FSParam[String]( name = "customized_retrieval_based_candidate_generation_offline_retweet_based_diffusion_model_id", default = ModelConfig.RetweetBasedDiffusion ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableOfflineInterestedInParam, EnableOfflineFTRInterestedInParam, EnableTwhinCollabFilterClusterParam, EnableTwhinMultiClusterParam, EnableRetweetBasedDiffusionParam, CustomizedRetrievalBasedRetweetDiffusionSource ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableOfflineInterestedInParam, EnableOfflineFTRInterestedInParam, EnableTwhinCollabFilterClusterParam, EnableTwhinMultiClusterParam, EnableRetweetBasedDiffusionParam ) val stringFSOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides( CustomizedRetrievalBasedRetweetDiffusionSource ) BaseConfigBuilder() .set(booleanOverrides: _*) .set(stringFSOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedFTROfflineInterestedInParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.cr_mixer.model.ModelConfig import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object CustomizedRetrievalBasedFTROfflineInterestedInParams { object CustomizedRetrievalBasedFTROfflineInterestedInSource extends FSParam[String]( name = "customized_retrieval_based_ftr_offline_interestedin_model_id", default = ModelConfig.OfflineFavDecayedSum ) val AllParams: Seq[Param[_] with FSName] = Seq( CustomizedRetrievalBasedFTROfflineInterestedInSource) lazy val config: BaseConfig = { val stringFSOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides( CustomizedRetrievalBasedFTROfflineInterestedInSource ) BaseConfigBuilder() .set(stringFSOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedOfflineInterestedInParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.cr_mixer.model.ModelConfig import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object CustomizedRetrievalBasedOfflineInterestedInParams { // Model slots available for offline InterestedIn candidate generation object CustomizedRetrievalBasedOfflineInterestedInSource extends FSParam[String]( name = "customized_retrieval_based_offline_interestedin_model_id", default = ModelConfig.OfflineInterestedInFromKnownFor2020 ) val AllParams: Seq[Param[_] with FSName] = Seq(CustomizedRetrievalBasedOfflineInterestedInSource) lazy val config: BaseConfig = { val stringFSOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides( CustomizedRetrievalBasedOfflineInterestedInSource ) BaseConfigBuilder() .set(stringFSOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedTwhinParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.cr_mixer.model.ModelConfig import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object CustomizedRetrievalBasedTwhinParams { // Model slots available for TwhinCollab and MultiCluster object CustomizedRetrievalBasedTwhinCollabFilterFollowSource extends FSParam[String]( name = "customized_retrieval_based_offline_twhin_collab_filter_follow_model_id", default = ModelConfig.TwhinCollabFilterForFollow ) object CustomizedRetrievalBasedTwhinCollabFilterEngagementSource extends FSParam[String]( name = "customized_retrieval_based_offline_twhin_collab_filter_engagement_model_id", default = ModelConfig.TwhinCollabFilterForEngagement ) object CustomizedRetrievalBasedTwhinMultiClusterFollowSource extends FSParam[String]( name = "customized_retrieval_based_offline_twhin_multi_cluster_follow_model_id", default = ModelConfig.TwhinMultiClusterForFollow ) object CustomizedRetrievalBasedTwhinMultiClusterEngagementSource extends FSParam[String]( name = "customized_retrieval_based_offline_twhin_multi_cluster_engagement_model_id", default = ModelConfig.TwhinMultiClusterForEngagement ) val AllParams: Seq[Param[_] with FSName] = Seq( CustomizedRetrievalBasedTwhinCollabFilterFollowSource, CustomizedRetrievalBasedTwhinCollabFilterEngagementSource, CustomizedRetrievalBasedTwhinMultiClusterFollowSource, CustomizedRetrievalBasedTwhinMultiClusterEngagementSource, ) lazy val config: BaseConfig = { val stringFSOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides( CustomizedRetrievalBasedTwhinCollabFilterFollowSource, CustomizedRetrievalBasedTwhinCollabFilterEngagementSource, CustomizedRetrievalBasedTwhinMultiClusterFollowSource, CustomizedRetrievalBasedTwhinMultiClusterEngagementSource, ) BaseConfigBuilder() .set(stringFSOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/EarlybirdFrsBasedCandidateGenerationParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.model.EarlybirdSimilarityEngineType import com.twitter.cr_mixer.model.EarlybirdSimilarityEngineType_ModelBased import com.twitter.cr_mixer.model.EarlybirdSimilarityEngineType_RecencyBased import com.twitter.cr_mixer.model.EarlybirdSimilarityEngineType_TensorflowBased import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.logging.Logger import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.timelines.configapi.Param import com.twitter.util.Duration object EarlybirdFrsBasedCandidateGenerationParams { object CandidateGenerationEarlybirdSimilarityEngineType extends Enumeration { protected case class SimilarityEngineType(rankingMode: EarlybirdSimilarityEngineType) extends super.Val import scala.language.implicitConversions implicit def valueToEarlybirdRankingMode(x: Value): SimilarityEngineType = x.asInstanceOf[SimilarityEngineType] val EarlybirdRankingMode_RecencyBased: SimilarityEngineType = SimilarityEngineType( EarlybirdSimilarityEngineType_RecencyBased) val EarlybirdRankingMode_ModelBased: SimilarityEngineType = SimilarityEngineType( EarlybirdSimilarityEngineType_ModelBased) val EarlybirdRankingMode_TensorflowBased: SimilarityEngineType = SimilarityEngineType( EarlybirdSimilarityEngineType_TensorflowBased) } object FrsBasedCandidateGenerationEarlybirdSimilarityEngineTypeParam extends FSEnumParam[CandidateGenerationEarlybirdSimilarityEngineType.type]( name = "frs_based_candidate_generation_earlybird_ranking_mode_id", default = CandidateGenerationEarlybirdSimilarityEngineType.EarlybirdRankingMode_RecencyBased, enum = CandidateGenerationEarlybirdSimilarityEngineType ) object FrsBasedCandidateGenerationRecencyBasedEarlybirdMaxTweetsPerUser extends FSBoundedParam[Int]( name = "frs_based_candidate_generation_earlybird_max_tweets_per_user", default = 100, min = 0, /** * Note max should be equal to EarlybirdRecencyBasedCandidateStoreModule.DefaultMaxNumTweetPerUser. * Which is the size of the memcached result list. */ max = 100 ) object FrsBasedCandidateGenerationEarlybirdMaxTweetAge extends FSBoundedParam[Duration]( name = "frs_based_candidate_generation_earlybird_max_tweet_age_hours", default = 24.hours, min = 12.hours, /** * Note max could be related to EarlybirdRecencyBasedCandidateStoreModule.DefaultMaxNumTweetPerUser. * Which is the size of the memcached result list for recency based earlybird candidate source. * E.g. if max = 720.hours, we may want to increase the DefaultMaxNumTweetPerUser. */ max = 96.hours ) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromHours } object FrsBasedCandidateGenerationEarlybirdFilterOutRetweetsAndReplies extends FSParam[Boolean]( name = "frs_based_candidate_generation_earlybird_filter_out_retweets_and_replies", default = true ) val AllParams: Seq[Param[_] with FSName] = Seq( FrsBasedCandidateGenerationEarlybirdSimilarityEngineTypeParam, FrsBasedCandidateGenerationRecencyBasedEarlybirdMaxTweetsPerUser, FrsBasedCandidateGenerationEarlybirdMaxTweetAge, FrsBasedCandidateGenerationEarlybirdFilterOutRetweetsAndReplies, ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( FrsBasedCandidateGenerationEarlybirdFilterOutRetweetsAndReplies, ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides() val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( FrsBasedCandidateGenerationRecencyBasedEarlybirdMaxTweetsPerUser ) val durationFSOverrides = FeatureSwitchOverrideUtil.getDurationFSOverrides( FrsBasedCandidateGenerationEarlybirdMaxTweetAge ) val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( NullStatsReceiver, Logger(getClass), FrsBasedCandidateGenerationEarlybirdSimilarityEngineTypeParam, ) BaseConfigBuilder() .set(booleanOverrides: _*) .set(doubleOverrides: _*) .set(intOverrides: _*) .set(enumOverrides: _*) .set(durationFSOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/FrsParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param import com.twitter.follow_recommendations.thriftscala.DisplayLocation import com.twitter.timelines.configapi.FSEnumParam import com.twitter.logging.Logger import com.twitter.finagle.stats.NullStatsReceiver object FrsParams { object EnableSourceParam extends FSParam[Boolean]( name = "signal_frs_enable_source", default = false ) object EnableSourceGraphParam extends FSParam[Boolean]( name = "graph_frs_enable_source", default = false ) object MinScoreParam extends FSBoundedParam[Double]( name = "signal_frs_min_score", default = 0.4, min = 0.0, max = 1.0 ) object MaxConsumerSeedsNumParam extends FSBoundedParam[Int]( name = "graph_frs_max_user_seeds_num", default = 200, min = 0, max = 1000 ) /** * These params below are only used for FrsTweetCandidateGenerator and shouldn't be used in other endpoints * * FrsBasedCandidateGenerationMaxSeedsNumParam * * FrsCandidateGenerationDisplayLocationParam * * FrsCandidateGenerationDisplayLocation * * FrsBasedCandidateGenerationMaxCandidatesNumParam */ object FrsBasedCandidateGenerationEnableVisibilityFilteringParam extends FSParam[Boolean]( name = "frs_based_candidate_generation_enable_vf", default = true ) object FrsBasedCandidateGenerationMaxSeedsNumParam extends FSBoundedParam[Int]( name = "frs_based_candidate_generation_max_seeds_num", default = 100, min = 0, max = 800 ) object FrsBasedCandidateGenerationDisplayLocation extends Enumeration { protected case class FrsDisplayLocationValue(displayLocation: DisplayLocation) extends super.Val import scala.language.implicitConversions implicit def valueToDisplayLocationValue(x: Value): FrsDisplayLocationValue = x.asInstanceOf[FrsDisplayLocationValue] val DisplayLocation_ContentRecommender: FrsDisplayLocationValue = FrsDisplayLocationValue( DisplayLocation.ContentRecommender) val DisplayLocation_Home: FrsDisplayLocationValue = FrsDisplayLocationValue( DisplayLocation.HomeTimelineTweetRecs) val DisplayLocation_Notifications: FrsDisplayLocationValue = FrsDisplayLocationValue( DisplayLocation.TweetNotificationRecs) } object FrsBasedCandidateGenerationDisplayLocationParam extends FSEnumParam[FrsBasedCandidateGenerationDisplayLocation.type]( name = "frs_based_candidate_generation_display_location_id", default = FrsBasedCandidateGenerationDisplayLocation.DisplayLocation_Home, enum = FrsBasedCandidateGenerationDisplayLocation ) object FrsBasedCandidateGenerationMaxCandidatesNumParam extends FSBoundedParam[Int]( name = "frs_based_candidate_generation_max_candidates_num", default = 100, min = 0, max = 2000 ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableSourceParam, EnableSourceGraphParam, MinScoreParam, MaxConsumerSeedsNumParam, FrsBasedCandidateGenerationMaxSeedsNumParam, FrsBasedCandidateGenerationDisplayLocationParam, FrsBasedCandidateGenerationMaxCandidatesNumParam, FrsBasedCandidateGenerationEnableVisibilityFilteringParam ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam, EnableSourceGraphParam, FrsBasedCandidateGenerationEnableVisibilityFilteringParam ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(MinScoreParam) val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( MaxConsumerSeedsNumParam, FrsBasedCandidateGenerationMaxSeedsNumParam, FrsBasedCandidateGenerationMaxCandidatesNumParam) val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( NullStatsReceiver, Logger(getClass), FrsBasedCandidateGenerationDisplayLocationParam, ) BaseConfigBuilder() .set(booleanOverrides: _*) .set(doubleOverrides: _*) .set(intOverrides: _*) .set(enumOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GlobalParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.logging.Logger import com.twitter.simclusters_v2.common.ModelVersions import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.timelines.configapi.Param import com.twitter.util.Duration /** * Instantiate Params that do not relate to a specific product. * The params in this file correspond to config repo file * [[https://sourcegraph.twitter.biz/config-git.twitter.biz/config/-/blob/features/cr-mixer/main/twistly_core.yml]] */ object GlobalParams { object MaxCandidatesPerRequestParam extends FSBoundedParam[Int]( name = "twistly_core_max_candidates_per_request", default = 100, min = 0, max = 9000 ) object ModelVersionParam extends FSEnumParam[ModelVersions.Enum.type]( name = "twistly_core_simclusters_model_version_id", default = ModelVersions.Enum.Model20M145K2020, enum = ModelVersions.Enum ) object UnifiedMaxSourceKeyNum extends FSBoundedParam[Int]( name = "twistly_core_unified_max_sourcekey_num", default = 15, min = 0, max = 100 ) object MaxCandidateNumPerSourceKeyParam extends FSBoundedParam[Int]( name = "twistly_core_candidate_per_sourcekey_max_num", default = 200, min = 0, max = 1000 ) // 1 hours to 30 days object MaxTweetAgeHoursParam extends FSBoundedParam[Duration]( name = "twistly_core_max_tweet_age_hours", default = 720.hours, min = 1.hours, max = 720.hours ) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromHours } val AllParams: Seq[Param[_] with FSName] = Seq( MaxCandidatesPerRequestParam, UnifiedMaxSourceKeyNum, MaxCandidateNumPerSourceKeyParam, ModelVersionParam, MaxTweetAgeHoursParam ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides() val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( MaxCandidatesPerRequestParam, UnifiedMaxSourceKeyNum, MaxCandidateNumPerSourceKeyParam ) val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( NullStatsReceiver, Logger(getClass), ModelVersionParam ) val boundedDurationFSOverrides = FeatureSwitchOverrideUtil.getBoundedDurationFSOverrides(MaxTweetAgeHoursParam) val seqOverrides = FeatureSwitchOverrideUtil.getLongSeqFSOverrides() BaseConfigBuilder() .set(booleanOverrides: _*) .set(intOverrides: _*) .set(boundedDurationFSOverrides: _*) .set(enumOverrides: _*) .set(seqOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GoodProfileClickParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.logging.Logger import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param import com.twitter.usersignalservice.thriftscala.SignalType object GoodProfileClickParams { object ClickMinDwellTimeParam extends Enumeration { protected case class SignalTypeValue(signalType: SignalType) extends super.Val import scala.language.implicitConversions implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = x.asInstanceOf[SignalTypeValue] val TotalDwellTime10s = SignalTypeValue(SignalType.GoodProfileClick) val TotalDwellTime20s = SignalTypeValue(SignalType.GoodProfileClick20s) val TotalDwellTime30s = SignalTypeValue(SignalType.GoodProfileClick30s) } object EnableSourceParam extends FSParam[Boolean]( name = "signal_good_profile_clicks_enable_source", default = false ) object ClickMinDwellTimeType extends FSEnumParam[ClickMinDwellTimeParam.type]( name = "signal_good_profile_clicks_min_dwelltime_type_id", default = ClickMinDwellTimeParam.TotalDwellTime10s, enum = ClickMinDwellTimeParam ) val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam, ClickMinDwellTimeType) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam ) val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( NullStatsReceiver, Logger(getClass), ClickMinDwellTimeType ) BaseConfigBuilder() .set(booleanOverrides: _*) .set(enumOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GoodTweetClickParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.logging.Logger import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param import com.twitter.usersignalservice.thriftscala.SignalType object GoodTweetClickParams { object ClickMinDwellTimeParam extends Enumeration { protected case class SignalTypeValue(signalType: SignalType) extends super.Val import scala.language.implicitConversions implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = x.asInstanceOf[SignalTypeValue] val TotalDwellTime2s = SignalTypeValue(SignalType.GoodTweetClick) val TotalDwellTime5s = SignalTypeValue(SignalType.GoodTweetClick5s) val TotalDwellTime10s = SignalTypeValue(SignalType.GoodTweetClick10s) val TotalDwellTime30s = SignalTypeValue(SignalType.GoodTweetClick30s) } object EnableSourceParam extends FSParam[Boolean]( name = "signal_good_tweet_clicks_enable_source", default = false ) object ClickMinDwellTimeType extends FSEnumParam[ClickMinDwellTimeParam.type]( name = "signal_good_tweet_clicks_min_dwelltime_type_id", default = ClickMinDwellTimeParam.TotalDwellTime2s, enum = ClickMinDwellTimeParam ) object MaxSignalNumParam extends FSBoundedParam[Int]( name = "signal_good_tweet_clicks_max_signal_num", default = 15, min = 0, max = 15 ) val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam, ClickMinDwellTimeType, MaxSignalNumParam) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam ) val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( NullStatsReceiver, Logger(getClass), ClickMinDwellTimeType ) val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( MaxSignalNumParam ) BaseConfigBuilder() .set(booleanOverrides: _*) .set(enumOverrides: _*) .set(intOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/InterestedInParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.logging.Logger import com.twitter.simclusters_v2.thriftscala.{EmbeddingType => SimClustersEmbeddingType} import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object InterestedInParams { object SourceEmbedding extends Enumeration { protected case class EmbeddingType(embeddingType: SimClustersEmbeddingType) extends super.Val import scala.language.implicitConversions implicit def valueToEmbeddingtype(x: Value): EmbeddingType = x.asInstanceOf[EmbeddingType] val UserInterestedIn: Value = EmbeddingType(SimClustersEmbeddingType.FilteredUserInterestedIn) val UnfilteredUserInterestedIn: Value = EmbeddingType( SimClustersEmbeddingType.UnfilteredUserInterestedIn) val FromProducerEmbedding: Value = EmbeddingType( SimClustersEmbeddingType.FilteredUserInterestedInFromPE) val LogFavBasedUserInterestedInFromAPE: Value = EmbeddingType( SimClustersEmbeddingType.LogFavBasedUserInterestedInFromAPE) val FollowBasedUserInterestedInFromAPE: Value = EmbeddingType( SimClustersEmbeddingType.FollowBasedUserInterestedInFromAPE) val UserNextInterestedIn: Value = EmbeddingType(SimClustersEmbeddingType.UserNextInterestedIn) // AddressBook based InterestedIn val LogFavBasedUserInterestedAverageAddressBookFromIIAPE: Value = EmbeddingType( SimClustersEmbeddingType.LogFavBasedUserInterestedAverageAddressBookFromIIAPE) val LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType( SimClustersEmbeddingType.LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE) val LogFavBasedUserInterestedBooktypeMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType( SimClustersEmbeddingType.LogFavBasedUserInterestedBooktypeMaxpoolingAddressBookFromIIAPE) val LogFavBasedUserInterestedLargestDimMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType( SimClustersEmbeddingType.LogFavBasedUserInterestedLargestDimMaxpoolingAddressBookFromIIAPE) val LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType( SimClustersEmbeddingType.LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE) val LogFavBasedUserInterestedConnectedMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType( SimClustersEmbeddingType.LogFavBasedUserInterestedConnectedMaxpoolingAddressBookFromIIAPE) } object EnableSourceParam extends FSParam[Boolean]( name = "twistly_interestedin_enable_source", default = true ) object InterestedInEmbeddingIdParam extends FSEnumParam[SourceEmbedding.type]( name = "twistly_interestedin_embedding_id", default = SourceEmbedding.UnfilteredUserInterestedIn, enum = SourceEmbedding ) object MinScoreParam extends FSBoundedParam[Double]( name = "twistly_interestedin_min_score", default = 0.072, min = 0.0, max = 1.0 ) object EnableSourceSequentialModelParam extends FSParam[Boolean]( name = "twistly_interestedin_sequential_model_enable_source", default = false ) object NextInterestedInEmbeddingIdParam extends FSEnumParam[SourceEmbedding.type]( name = "twistly_interestedin_sequential_model_embedding_id", default = SourceEmbedding.UserNextInterestedIn, enum = SourceEmbedding ) object MinScoreSequentialModelParam extends FSBoundedParam[Double]( name = "twistly_interestedin_sequential_model_min_score", default = 0.0, min = 0.0, max = 1.0 ) object EnableSourceAddressBookParam extends FSParam[Boolean]( name = "twistly_interestedin_addressbook_enable_source", default = false ) object AddressBookInterestedInEmbeddingIdParam extends FSEnumParam[SourceEmbedding.type]( name = "twistly_interestedin_addressbook_embedding_id", default = SourceEmbedding.LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE, enum = SourceEmbedding ) object MinScoreAddressBookParam extends FSBoundedParam[Double]( name = "twistly_interestedin_addressbook_min_score", default = 0.0, min = 0.0, max = 1.0 ) // Prod SimClusters ANN param // This is used to enable/disable querying of production SANN service. Useful when experimenting // with replacements to it. object EnableProdSimClustersANNParam extends FSParam[Boolean]( name = "twistly_interestedin_enable_prod_simclusters_ann", default = true ) // Experimental SimClusters ANN params object EnableExperimentalSimClustersANNParam extends FSParam[Boolean]( name = "twistly_interestedin_enable_experimental_simclusters_ann", default = false ) // SimClusters ANN 1 cluster params object EnableSimClustersANN1Param extends FSParam[Boolean]( name = "twistly_interestedin_enable_simclusters_ann_1", default = false ) // SimClusters ANN 2 cluster params object EnableSimClustersANN2Param extends FSParam[Boolean]( name = "twistly_interestedin_enable_simclusters_ann_2", default = false ) // SimClusters ANN 3 cluster params object EnableSimClustersANN3Param extends FSParam[Boolean]( name = "twistly_interestedin_enable_simclusters_ann_3", default = false ) // SimClusters ANN 5 cluster params object EnableSimClustersANN5Param extends FSParam[Boolean]( name = "twistly_interestedin_enable_simclusters_ann_5", default = false ) // SimClusters ANN 4 cluster params object EnableSimClustersANN4Param extends FSParam[Boolean]( name = "twistly_interestedin_enable_simclusters_ann_4", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableSourceParam, EnableSourceSequentialModelParam, EnableSourceAddressBookParam, EnableProdSimClustersANNParam, EnableExperimentalSimClustersANNParam, EnableSimClustersANN1Param, EnableSimClustersANN2Param, EnableSimClustersANN3Param, EnableSimClustersANN5Param, EnableSimClustersANN4Param, MinScoreParam, MinScoreSequentialModelParam, MinScoreAddressBookParam, InterestedInEmbeddingIdParam, NextInterestedInEmbeddingIdParam, AddressBookInterestedInEmbeddingIdParam, ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam, EnableSourceSequentialModelParam, EnableSourceAddressBookParam, EnableProdSimClustersANNParam, EnableExperimentalSimClustersANNParam, EnableSimClustersANN1Param, EnableSimClustersANN2Param, EnableSimClustersANN3Param, EnableSimClustersANN5Param, EnableSimClustersANN4Param ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( MinScoreParam, MinScoreSequentialModelParam, MinScoreAddressBookParam) val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( NullStatsReceiver, Logger(getClass), InterestedInEmbeddingIdParam, NextInterestedInEmbeddingIdParam, AddressBookInterestedInEmbeddingIdParam ) BaseConfigBuilder() .set(booleanOverrides: _*) .set(doubleOverrides: _*) .set(enumOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedCandidateGenerationParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.logging.Logger import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object ProducerBasedCandidateGenerationParams { // Source params. Not being used. It is always set to true in prod object EnableSourceParam extends FSParam[Boolean]( name = "producer_based_candidate_generation_enable_source", default = false ) object UtgCombinationMethodParam extends FSEnumParam[UnifiedSETweetCombinationMethod.type]( name = "producer_based_candidate_generation_utg_combination_method_id", default = UnifiedSETweetCombinationMethod.Frontload, enum = UnifiedSETweetCombinationMethod ) // UTG params object EnableUTGParam extends FSParam[Boolean]( name = "producer_based_candidate_generation_enable_utg", default = false ) object EnableUAGParam extends FSParam[Boolean]( name = "producer_based_candidate_generation_enable_uag", default = false ) // SimClusters params object EnableSimClustersANNParam extends FSParam[Boolean]( name = "producer_based_candidate_generation_enable_simclusters", default = true ) // Filter params object SimClustersMinScoreParam extends FSBoundedParam[Double]( name = "producer_based_candidate_generation_filter_simclusters_min_score", default = 0.7, min = 0.0, max = 1.0 ) // Experimental SimClusters ANN params object EnableExperimentalSimClustersANNParam extends FSParam[Boolean]( name = "producer_based_candidate_generation_enable_experimental_simclusters_ann", default = false ) // SimClusters ANN cluster 1 params object EnableSimClustersANN1Param extends FSParam[Boolean]( name = "producer_based_candidate_generation_enable_simclusters_ann_1", default = false ) // SimClusters ANN cluster 2 params object EnableSimClustersANN2Param extends FSParam[Boolean]( name = "producer_based_candidate_generation_enable_simclusters_ann_2", default = false ) // SimClusters ANN cluster 3 params object EnableSimClustersANN3Param extends FSParam[Boolean]( name = "producer_based_candidate_generation_enable_simclusters_ann_3", default = false ) // SimClusters ANN cluster 5 params object EnableSimClustersANN5Param extends FSParam[Boolean]( name = "producer_based_candidate_generation_enable_simclusters_ann_5", default = false ) object EnableSimClustersANN4Param extends FSParam[Boolean]( name = "producer_based_candidate_generation_enable_simclusters_ann_4", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableSourceParam, EnableUAGParam, EnableUTGParam, EnableSimClustersANNParam, EnableSimClustersANN1Param, EnableSimClustersANN2Param, EnableSimClustersANN3Param, EnableSimClustersANN5Param, EnableSimClustersANN4Param, EnableExperimentalSimClustersANNParam, SimClustersMinScoreParam, UtgCombinationMethodParam ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam, EnableUAGParam, EnableUTGParam, EnableSimClustersANNParam, EnableSimClustersANN1Param, EnableSimClustersANN2Param, EnableSimClustersANN3Param, EnableSimClustersANN5Param, EnableSimClustersANN4Param, EnableExperimentalSimClustersANNParam ) val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( NullStatsReceiver, Logger(getClass), UtgCombinationMethodParam, ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(SimClustersMinScoreParam) BaseConfigBuilder() .set(booleanOverrides: _*) .set(doubleOverrides: _*) .set(enumOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedUserAdGraphParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object ProducerBasedUserAdGraphParams { object MinCoOccurrenceParam extends FSBoundedParam[Int]( name = "producer_based_user_ad_graph_min_co_occurrence", default = 2, min = 0, max = 500 ) object MinScoreParam extends FSBoundedParam[Double]( name = "producer_based_user_ad_graph_min_score", default = 3.0, min = 0.0, max = 10.0 ) object MaxNumFollowersParam extends FSBoundedParam[Int]( name = "producer_based_user_ad_graph_max_num_followers", default = 500, min = 100, max = 1000 ) val AllParams: Seq[Param[_] with FSName] = Seq(MinCoOccurrenceParam, MaxNumFollowersParam, MinScoreParam) lazy val config: BaseConfig = { val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( MinCoOccurrenceParam, MaxNumFollowersParam, ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(MinScoreParam) BaseConfigBuilder() .set(intOverrides: _*) .set(doubleOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedUserTweetGraphParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object ProducerBasedUserTweetGraphParams { object MinCoOccurrenceParam extends FSBoundedParam[Int]( name = "producer_based_user_tweet_graph_min_co_occurrence", default = 4, min = 0, max = 500 ) object MinScoreParam extends FSBoundedParam[Double]( name = "producer_based_user_tweet_graph_min_score", default = 3.0, min = 0.0, max = 10.0 ) object MaxNumFollowersParam extends FSBoundedParam[Int]( name = "producer_based_user_tweet_graph_max_num_followers", default = 500, min = 100, max = 1000 ) val AllParams: Seq[Param[_] with FSName] = Seq(MinCoOccurrenceParam, MaxNumFollowersParam, MinScoreParam) lazy val config: BaseConfig = { val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( MinCoOccurrenceParam, MaxNumFollowersParam, ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(MinScoreParam) BaseConfigBuilder() .set(intOverrides: _*) .set(doubleOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RankerParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.logging.Logger import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object RankerParams { object MaxCandidatesToRank extends FSBoundedParam[Int]( name = "twistly_core_max_candidates_to_rank", default = 2000, min = 0, max = 9999 ) object EnableBlueVerifiedTopK extends FSParam[Boolean]( name = "twistly_core_blue_verified_top_k", default = true ) val AllParams: Seq[Param[_] with FSName] = Seq( MaxCandidatesToRank, EnableBlueVerifiedTopK ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides(EnableBlueVerifiedTopK) val boundedDurationFSOverrides = FeatureSwitchOverrideUtil.getBoundedDurationFSOverrides() val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( MaxCandidatesToRank ) val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( NullStatsReceiver, Logger(getClass), ) val stringFSOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides() BaseConfigBuilder() .set(booleanOverrides: _*) .set(boundedDurationFSOverrides: _*) .set(intOverrides: _*) .set(enumOverrides: _*) .set(stringFSOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RealGraphInParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi._ object RealGraphInParams { object EnableSourceGraphParam extends FSParam[Boolean]( name = "graph_realgraphin_enable_source", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableSourceGraphParam, ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceGraphParam ) BaseConfigBuilder() .set(booleanOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RealGraphOonParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object RealGraphOonParams { object EnableSourceParam extends FSParam[Boolean]( name = "signal_realgraphoon_enable_source", default = false ) object EnableSourceGraphParam extends FSParam[Boolean]( name = "graph_realgraphoon_enable_source", default = false ) object MaxConsumerSeedsNumParam extends FSBoundedParam[Int]( name = "graph_realgraphoon_max_user_seeds_num", default = 200, min = 0, max = 1000 ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableSourceParam, EnableSourceGraphParam, MaxConsumerSeedsNumParam ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam, EnableSourceGraphParam ) val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides(MaxConsumerSeedsNumParam) BaseConfigBuilder() .set(booleanOverrides: _*) .set(intOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentFollowsParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object RecentFollowsParams { object EnableSourceParam extends FSParam[Boolean]( name = "twistly_recentfollows_enable_source", default = true ) val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam ) BaseConfigBuilder() .set(booleanOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentNegativeSignalParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.logging.Logger import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object RecentNegativeSignalParams { object EnableSourceParam extends FSParam[Boolean]( name = "twistly_recentnegativesignals_enable_source", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableSourceParam ) lazy val config: BaseConfig = { val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( NullStatsReceiver, Logger(getClass), ) val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides() BaseConfigBuilder() .set(booleanOverrides: _*).set(doubleOverrides: _*).set(enumOverrides: _*).build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentNotificationsParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object RecentNotificationsParams { object EnableSourceParam extends FSParam[Boolean]( name = "twistly_recentnotifications_enable_source", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam ) BaseConfigBuilder() .set(booleanOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentOriginalTweetsParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object RecentOriginalTweetsParams { // Source params object EnableSourceParam extends FSParam[Boolean]( name = "twistly_recentoriginaltweets_enable_source", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides(EnableSourceParam) BaseConfigBuilder() .set(booleanOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentReplyTweetsParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object RecentReplyTweetsParams { // Source params object EnableSourceParam extends FSParam[Boolean]( name = "twistly_recentreplytweets_enable_source", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides(EnableSourceParam) BaseConfigBuilder() .set(booleanOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentRetweetsParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object RecentRetweetsParams { // Source params object EnableSourceParam extends FSParam[Boolean]( name = "twistly_recentretweets_enable_source", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam ) BaseConfigBuilder() .set(booleanOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentTweetFavoritesParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object RecentTweetFavoritesParams { // Source params object EnableSourceParam extends FSParam[Boolean]( name = "twistly_recenttweetfavorites_enable_source", default = true ) val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam ) BaseConfigBuilder() .set(booleanOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetGlobalParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object RelatedTweetGlobalParams { object MaxCandidatesPerRequestParam extends FSBoundedParam[Int]( name = "related_tweet_core_max_candidates_per_request", default = 100, min = 0, max = 500 ) val AllParams: Seq[Param[_] with FSName] = Seq(MaxCandidatesPerRequestParam) lazy val config: BaseConfig = { val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( MaxCandidatesPerRequestParam ) BaseConfigBuilder() .set(intOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetProducerBasedParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object RelatedTweetProducerBasedParams { // UTG params object EnableUTGParam extends FSParam[Boolean]( name = "related_tweet_producer_based_enable_utg", default = false ) // SimClusters params object EnableSimClustersANNParam extends FSParam[Boolean]( name = "related_tweet_producer_based_enable_simclusters", default = true ) // Filter params object SimClustersMinScoreParam extends FSBoundedParam[Double]( name = "related_tweet_producer_based_filter_simclusters_min_score", default = 0.0, min = 0.0, max = 1.0 ) // Experimental SimClusters ANN params object EnableExperimentalSimClustersANNParam extends FSParam[Boolean]( name = "related_tweet_producer_based_enable_experimental_simclusters_ann", default = false ) // SimClusters ANN cluster 1 params object EnableSimClustersANN1Param extends FSParam[Boolean]( name = "related_tweet_producer_based_enable_simclusters_ann_1", default = false ) // SimClusters ANN cluster 2 params object EnableSimClustersANN2Param extends FSParam[Boolean]( name = "related_tweet_producer_based_enable_simclusters_ann_2", default = false ) // SimClusters ANN cluster 3 params object EnableSimClustersANN3Param extends FSParam[Boolean]( name = "related_tweet_producer_based_enable_simclusters_ann_3", default = false ) // SimClusters ANN cluster 3 params object EnableSimClustersANN5Param extends FSParam[Boolean]( name = "related_tweet_producer_based_enable_simclusters_ann_5", default = false ) // SimClusters ANN cluster 4 params object EnableSimClustersANN4Param extends FSParam[Boolean]( name = "related_tweet_producer_based_enable_simclusters_ann_4", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableUTGParam, EnableSimClustersANNParam, EnableSimClustersANN1Param, EnableSimClustersANN2Param, EnableSimClustersANN3Param, EnableSimClustersANN5Param, EnableSimClustersANN4Param, EnableExperimentalSimClustersANNParam, SimClustersMinScoreParam ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableUTGParam, EnableSimClustersANNParam, EnableSimClustersANN1Param, EnableSimClustersANN2Param, EnableSimClustersANN3Param, EnableSimClustersANN5Param, EnableSimClustersANN4Param, EnableExperimentalSimClustersANNParam ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( SimClustersMinScoreParam ) BaseConfigBuilder() .set(booleanOverrides: _*) .set(doubleOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetTweetBasedParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object RelatedTweetTweetBasedParams { // UTG params object EnableUTGParam extends FSParam[Boolean]( name = "related_tweet_tweet_based_enable_utg", default = false ) // UVG params object EnableUVGParam extends FSParam[Boolean]( name = "related_tweet_tweet_based_enable_uvg", default = false ) // UAG params object EnableUAGParam extends FSParam[Boolean]( name = "related_tweet_tweet_based_enable_uag", default = false ) // SimClusters params object EnableSimClustersANNParam extends FSParam[Boolean]( name = "related_tweet_tweet_based_enable_simclusters", default = true ) // Experimental SimClusters ANN params object EnableExperimentalSimClustersANNParam extends FSParam[Boolean]( name = "related_tweet_tweet_based_enable_experimental_simclusters_ann", default = false ) // SimClusters ANN cluster 1 params object EnableSimClustersANN1Param extends FSParam[Boolean]( name = "related_tweet_tweet_based_enable_simclusters_ann_1", default = false ) // SimClusters ANN cluster 2 params object EnableSimClustersANN2Param extends FSParam[Boolean]( name = "related_tweet_tweet_based_enable_simclusters_ann_2", default = false ) // SimClusters ANN cluster 3 params object EnableSimClustersANN3Param extends FSParam[Boolean]( name = "related_tweet_tweet_based_enable_simclusters_ann_3", default = false ) // SimClusters ANN cluster 5 params object EnableSimClustersANN5Param extends FSParam[Boolean]( name = "related_tweet_tweet_based_enable_simclusters_ann_5", default = false ) object EnableSimClustersANN4Param extends FSParam[Boolean]( name = "related_tweet_tweet_based_enable_simclusters_ann_4", default = false ) // TwHIN params object EnableTwHINParam extends FSParam[Boolean]( name = "related_tweet_tweet_based_enable_twhin", default = false ) // QIG params object EnableQigSimilarTweetsParam extends FSParam[Boolean]( name = "related_tweet_tweet_based_enable_qig_similar_tweets", default = false ) // Filter params object SimClustersMinScoreParam extends FSBoundedParam[Double]( name = "related_tweet_tweet_based_filter_simclusters_min_score", default = 0.3, min = 0.0, max = 1.0 ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableTwHINParam, EnableQigSimilarTweetsParam, EnableUTGParam, EnableUVGParam, EnableSimClustersANNParam, EnableSimClustersANN2Param, EnableSimClustersANN3Param, EnableSimClustersANN5Param, EnableSimClustersANN4Param, EnableExperimentalSimClustersANNParam, SimClustersMinScoreParam ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableTwHINParam, EnableQigSimilarTweetsParam, EnableUTGParam, EnableUVGParam, EnableSimClustersANNParam, EnableSimClustersANN2Param, EnableSimClustersANN3Param, EnableSimClustersANN5Param, EnableSimClustersANN4Param, EnableExperimentalSimClustersANNParam ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(SimClustersMinScoreParam) BaseConfigBuilder() .set(booleanOverrides: _*) .set(doubleOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedVideoTweetGlobalParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object RelatedVideoTweetGlobalParams { object MaxCandidatesPerRequestParam extends FSBoundedParam[Int]( name = "related_video_tweet_core_max_candidates_per_request", default = 100, min = 0, max = 500 ) val AllParams: Seq[Param[_] with FSName] = Seq(MaxCandidatesPerRequestParam) lazy val config: BaseConfig = { val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( MaxCandidatesPerRequestParam ) BaseConfigBuilder() .set(intOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedVideoTweetTweetBasedParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object RelatedVideoTweetTweetBasedParams { // UTG params object EnableUTGParam extends FSParam[Boolean]( name = "related_video_tweet_tweet_based_enable_utg", default = false ) // SimClusters params object EnableSimClustersANNParam extends FSParam[Boolean]( name = "related_video_tweet_tweet_based_enable_simclusters", default = true ) // Experimental SimClusters ANN params object EnableExperimentalSimClustersANNParam extends FSParam[Boolean]( name = "related_video_tweet_tweet_based_enable_experimental_simclusters_ann", default = false ) // SimClusters ANN cluster 1 params object EnableSimClustersANN1Param extends FSParam[Boolean]( name = "related_video_tweet_tweet_based_enable_simclusters_ann_1", default = false ) // SimClusters ANN cluster 2 params object EnableSimClustersANN2Param extends FSParam[Boolean]( name = "related_video_tweet_tweet_based_enable_simclusters_ann_2", default = false ) // SimClusters ANN cluster 3 params object EnableSimClustersANN3Param extends FSParam[Boolean]( name = "related_video_tweet_tweet_based_enable_simclusters_ann_3", default = false ) // SimClusters ANN cluster 5 params object EnableSimClustersANN5Param extends FSParam[Boolean]( name = "related_video_tweet_tweet_based_enable_simclusters_ann_5", default = false ) // SimClusters ANN cluster 4 params object EnableSimClustersANN4Param extends FSParam[Boolean]( name = "related_video_tweet_tweet_based_enable_simclusters_ann_4", default = false ) // TwHIN params object EnableTwHINParam extends FSParam[Boolean]( name = "related_video_tweet_tweet_based_enable_twhin", default = false ) // QIG params object EnableQigSimilarTweetsParam extends FSParam[Boolean]( name = "related_video_tweet_tweet_based_enable_qig_similar_tweets", default = false ) // Filter params object SimClustersMinScoreParam extends FSBoundedParam[Double]( name = "related_video_tweet_tweet_based_filter_simclusters_min_score", default = 0.3, min = 0.0, max = 1.0 ) object EnableUVGParam extends FSParam[Boolean]( name = "related_video_tweet_tweet_based_enable_uvg", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableTwHINParam, EnableQigSimilarTweetsParam, EnableUTGParam, EnableUVGParam, EnableSimClustersANNParam, EnableSimClustersANN2Param, EnableSimClustersANN3Param, EnableSimClustersANN5Param, EnableSimClustersANN4Param, EnableExperimentalSimClustersANNParam, SimClustersMinScoreParam ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableTwHINParam, EnableQigSimilarTweetsParam, EnableUTGParam, EnableUVGParam, EnableSimClustersANNParam, EnableSimClustersANN2Param, EnableSimClustersANN3Param, EnableSimClustersANN5Param, EnableSimClustersANN4Param, EnableExperimentalSimClustersANNParam ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(SimClustersMinScoreParam) BaseConfigBuilder() .set(booleanOverrides: _*) .set(doubleOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RepeatedProfileVisitsParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.logging.Logger import com.twitter.usersignalservice.thriftscala.SignalType import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object RepeatedProfileVisitsParams { object ProfileMinVisitParam extends Enumeration { protected case class SignalTypeValue(signalType: SignalType) extends super.Val import scala.language.implicitConversions implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = x.asInstanceOf[SignalTypeValue] val TotalVisitsInPast180Days = SignalTypeValue(SignalType.RepeatedProfileVisit180dMinVisit6V1) val TotalVisitsInPast90Days = SignalTypeValue(SignalType.RepeatedProfileVisit90dMinVisit6V1) val TotalVisitsInPast14Days = SignalTypeValue(SignalType.RepeatedProfileVisit14dMinVisit2V1) val TotalVisitsInPast180DaysNoNegative = SignalTypeValue( SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative) val TotalVisitsInPast90DaysNoNegative = SignalTypeValue( SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative) val TotalVisitsInPast14DaysNoNegative = SignalTypeValue( SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative) } object EnableSourceParam extends FSParam[Boolean]( name = "twistly_repeatedprofilevisits_enable_source", default = true ) object MinScoreParam extends FSBoundedParam[Double]( name = "twistly_repeatedprofilevisits_min_score", default = 0.5, min = 0.0, max = 1.0 ) object ProfileMinVisitType extends FSEnumParam[ProfileMinVisitParam.type]( name = "twistly_repeatedprofilevisits_min_visit_type_id", default = ProfileMinVisitParam.TotalVisitsInPast14Days, enum = ProfileMinVisitParam ) val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam, ProfileMinVisitType) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam ) val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( NullStatsReceiver, Logger(getClass), ProfileMinVisitType ) BaseConfigBuilder() .set(booleanOverrides: _*) .set(enumOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/SimClustersANNParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object SimClustersANNParams { // Different SimClusters ANN cluster has its own config id (model slot) object SimClustersANNConfigId extends FSParam[String]( name = "similarity_simclusters_ann_simclusters_ann_config_id", default = "Default" ) object SimClustersANN1ConfigId extends FSParam[String]( name = "similarity_simclusters_ann_simclusters_ann_1_config_id", default = "20220810" ) object SimClustersANN2ConfigId extends FSParam[String]( name = "similarity_simclusters_ann_simclusters_ann_2_config_id", default = "20220818" ) object SimClustersANN3ConfigId extends FSParam[String]( name = "similarity_simclusters_ann_simclusters_ann_3_config_id", default = "20220819" ) object SimClustersANN5ConfigId extends FSParam[String]( name = "similarity_simclusters_ann_simclusters_ann_5_config_id", default = "20221221" ) object SimClustersANN4ConfigId extends FSParam[String]( name = "similarity_simclusters_ann_simclusters_ann_4_config_id", default = "20221220" ) object ExperimentalSimClustersANNConfigId extends FSParam[String]( name = "similarity_simclusters_ann_experimental_simclusters_ann_config_id", default = "20220801" ) val AllParams: Seq[Param[_] with FSName] = Seq( SimClustersANNConfigId, SimClustersANN1ConfigId, SimClustersANN2ConfigId, SimClustersANN3ConfigId, SimClustersANN5ConfigId, ExperimentalSimClustersANNConfigId ) lazy val config: BaseConfig = { val stringOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides( SimClustersANNConfigId, SimClustersANN1ConfigId, SimClustersANN2ConfigId, SimClustersANN3ConfigId, SimClustersANN5ConfigId, ExperimentalSimClustersANNConfigId ) BaseConfigBuilder() .set(stringOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TopicTweetParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.logging.Logger import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.timelines.configapi.Param import com.twitter.util.Duration object TopicTweetParams { object MaxTweetAge extends FSBoundedParam[Duration]( name = "topic_tweet_candidate_generation_max_tweet_age_hours", default = 24.hours, min = 12.hours, max = 48.hours ) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromHours } object MaxTopicTweetCandidatesParam extends FSBoundedParam[Int]( name = "topic_tweet_max_candidates_num", default = 200, min = 0, max = 1000 ) object MaxSkitTfgCandidatesParam extends FSBoundedParam[Int]( name = "topic_tweet_skit_tfg_max_candidates_num", default = 100, min = 0, max = 1000 ) object MaxSkitHighPrecisionCandidatesParam extends FSBoundedParam[Int]( name = "topic_tweet_skit_high_precision_max_candidates_num", default = 100, min = 0, max = 1000 ) object MaxCertoCandidatesParam extends FSBoundedParam[Int]( name = "topic_tweet_certo_max_candidates_num", default = 100, min = 0, max = 1000 ) // The min prod score for Certo L2-normalized cosine candidates object CertoScoreThresholdParam extends FSBoundedParam[Double]( name = "topic_tweet_certo_score_threshold", default = 0.015, min = 0, max = 1 ) object SemanticCoreVersionIdParam extends FSParam[Long]( name = "semantic_core_version_id", default = 1380520918896713735L ) val AllParams: Seq[Param[_] with FSName] = Seq( CertoScoreThresholdParam, MaxTopicTweetCandidatesParam, MaxTweetAge, MaxCertoCandidatesParam, MaxSkitTfgCandidatesParam, MaxSkitHighPrecisionCandidatesParam, SemanticCoreVersionIdParam ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides() val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(CertoScoreThresholdParam) val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( MaxCertoCandidatesParam, MaxSkitTfgCandidatesParam, MaxSkitHighPrecisionCandidatesParam, MaxTopicTweetCandidatesParam ) val longOverrides = FeatureSwitchOverrideUtil.getLongFSOverrides(SemanticCoreVersionIdParam) val durationFSOverrides = FeatureSwitchOverrideUtil.getDurationFSOverrides(MaxTweetAge) val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides(NullStatsReceiver, Logger(getClass)) BaseConfigBuilder() .set(booleanOverrides: _*) .set(doubleOverrides: _*) .set(intOverrides: _*) .set(longOverrides: _*) .set(enumOverrides: _*) .set(durationFSOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedCandidateGenerationParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.logging.Logger import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object TweetBasedCandidateGenerationParams { // Source params. Not being used. It is always set to true in prod object EnableSourceParam extends FSParam[Boolean]( name = "tweet_based_candidate_generation_enable_source", default = false ) // UTG params object EnableUTGParam extends FSParam[Boolean]( name = "tweet_based_candidate_generation_enable_utg", default = true ) // SimClusters params object EnableSimClustersANNParam extends FSParam[Boolean]( name = "tweet_based_candidate_generation_enable_simclusters", default = true ) // Experimental SimClusters ANN params object EnableExperimentalSimClustersANNParam extends FSParam[Boolean]( name = "tweet_based_candidate_generation_enable_experimental_simclusters_ann", default = false ) // SimClusters ANN cluster 1 params object EnableSimClustersANN1Param extends FSParam[Boolean]( name = "tweet_based_candidate_generation_enable_simclusters_ann_1", default = false ) // SimClusters ANN cluster 2 params object EnableSimClustersANN2Param extends FSParam[Boolean]( name = "tweet_based_candidate_generation_enable_simclusters_ann_2", default = false ) // SimClusters ANN cluster 3 params object EnableSimClustersANN3Param extends FSParam[Boolean]( name = "tweet_based_candidate_generation_enable_simclusters_ann_3", default = false ) // SimClusters ANN cluster 3 params object EnableSimClustersANN5Param extends FSParam[Boolean]( name = "tweet_based_candidate_generation_enable_simclusters_ann_5", default = false ) // SimClusters ANN cluster 4 params object EnableSimClustersANN4Param extends FSParam[Boolean]( name = "tweet_based_candidate_generation_enable_simclusters_ann_4", default = false ) // TwHIN params object EnableTwHINParam extends FSParam[Boolean]( name = "tweet_based_candidate_generation_enable_twhin", default = false ) // QIG params object EnableQigSimilarTweetsParam extends FSParam[Boolean]( name = "tweet_based_candidate_generation_enable_qig_similar_tweets", default = false ) object QigMaxNumSimilarTweetsParam extends FSBoundedParam[Int]( name = "tweet_based_candidate_generation_qig_max_num_similar_tweets", default = 100, min = 10, max = 100 ) // UVG params object EnableUVGParam extends FSParam[Boolean]( name = "tweet_based_candidate_generation_enable_uvg", default = false ) // UAG params object EnableUAGParam extends FSParam[Boolean]( name = "tweet_based_candidate_generation_enable_uag", default = false ) // Filter params object SimClustersMinScoreParam extends FSBoundedParam[Double]( name = "tweet_based_candidate_generation_filter_simclusters_min_score", default = 0.5, min = 0.0, max = 1.0 ) // for learning DDG that has a higher threshold for video based SANN object SimClustersVideoBasedMinScoreParam extends FSBoundedParam[Double]( name = "tweet_based_candidate_generation_filter_simclusters_video_based_min_score", default = 0.5, min = 0.0, max = 1.0 ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableSourceParam, EnableTwHINParam, EnableQigSimilarTweetsParam, EnableUTGParam, EnableUVGParam, EnableUAGParam, EnableSimClustersANNParam, EnableSimClustersANN1Param, EnableSimClustersANN2Param, EnableSimClustersANN3Param, EnableSimClustersANN5Param, EnableSimClustersANN4Param, EnableExperimentalSimClustersANNParam, SimClustersMinScoreParam, SimClustersVideoBasedMinScoreParam, QigMaxNumSimilarTweetsParam, ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam, EnableTwHINParam, EnableQigSimilarTweetsParam, EnableUTGParam, EnableUVGParam, EnableUAGParam, EnableSimClustersANNParam, EnableSimClustersANN1Param, EnableSimClustersANN2Param, EnableSimClustersANN3Param, EnableSimClustersANN5Param, EnableSimClustersANN4Param, EnableExperimentalSimClustersANNParam, ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( SimClustersMinScoreParam, SimClustersVideoBasedMinScoreParam) val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( NullStatsReceiver, Logger(getClass), ) val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( QigMaxNumSimilarTweetsParam ) BaseConfigBuilder() .set(booleanOverrides: _*) .set(doubleOverrides: _*) .set(enumOverrides: _*) .set(intOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedTwHINParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.cr_mixer.model.ModelConfig import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object TweetBasedTwHINParams { object ModelIdParam extends FSParam[String]( name = "tweet_based_twhin_model_id", default = ModelConfig.TweetBasedTwHINRegularUpdateAll20221024, ) val AllParams: Seq[Param[_] with FSName] = Seq(ModelIdParam) lazy val config: BaseConfig = { val stringFSOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides( ModelIdParam ) BaseConfigBuilder() .set(stringFSOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserAdGraphParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object TweetBasedUserAdGraphParams { object MinCoOccurrenceParam extends FSBoundedParam[Int]( name = "tweet_based_user_ad_graph_min_co_occurrence", default = 1, min = 0, max = 500 ) object ConsumersBasedMinScoreParam extends FSBoundedParam[Double]( name = "tweet_based_user_ad_graph_consumers_based_min_score", default = 0.0, min = 0.0, max = 10.0 ) object MaxConsumerSeedsNumParam extends FSBoundedParam[Int]( name = "tweet_based_user_ad_graph_max_user_seeds_num", default = 100, min = 0, max = 300 ) val AllParams: Seq[Param[_] with FSName] = Seq( MinCoOccurrenceParam, MaxConsumerSeedsNumParam, ConsumersBasedMinScoreParam ) lazy val config: BaseConfig = { val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( MinCoOccurrenceParam, MaxConsumerSeedsNumParam ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(ConsumersBasedMinScoreParam) BaseConfigBuilder() .set(intOverrides: _*) .set(doubleOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserTweetGraphParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object TweetBasedUserTweetGraphParams { object MinCoOccurrenceParam extends FSBoundedParam[Int]( name = "tweet_based_user_tweet_graph_min_co_occurrence", default = 3, min = 0, max = 500 ) object TweetBasedMinScoreParam extends FSBoundedParam[Double]( name = "tweet_based_user_tweet_graph_tweet_based_min_score", default = 0.5, min = 0.0, max = 10.0 ) object ConsumersBasedMinScoreParam extends FSBoundedParam[Double]( name = "tweet_based_user_tweet_graph_consumers_based_min_score", default = 4.0, min = 0.0, max = 10.0 ) object MaxConsumerSeedsNumParam extends FSBoundedParam[Int]( name = "tweet_based_user_tweet_graph_max_user_seeds_num", default = 100, min = 0, max = 300 ) object EnableCoverageExpansionOldTweetParam extends FSParam[Boolean]( name = "tweet_based_user_tweet_graph_enable_coverage_expansion_old_tweet", default = false ) object EnableCoverageExpansionAllTweetParam extends FSParam[Boolean]( name = "tweet_based_user_tweet_graph_enable_coverage_expansion_all_tweet", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableCoverageExpansionAllTweetParam, EnableCoverageExpansionOldTweetParam, MinCoOccurrenceParam, MaxConsumerSeedsNumParam, TweetBasedMinScoreParam, ConsumersBasedMinScoreParam ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableCoverageExpansionAllTweetParam, EnableCoverageExpansionOldTweetParam ) val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( MinCoOccurrenceParam, MaxConsumerSeedsNumParam ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( TweetBasedMinScoreParam, ConsumersBasedMinScoreParam) BaseConfigBuilder() .set(booleanOverrides: _*) .set(intOverrides: _*) .set(doubleOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserVideoGraphParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object TweetBasedUserVideoGraphParams { object MinCoOccurrenceParam extends FSBoundedParam[Int]( name = "tweet_based_user_video_graph_min_co_occurrence", default = 5, min = 0, max = 500 ) object TweetBasedMinScoreParam extends FSBoundedParam[Double]( name = "tweet_based_user_video_graph_tweet_based_min_score", default = 0.0, min = 0.0, max = 100.0 ) object ConsumersBasedMinScoreParam extends FSBoundedParam[Double]( name = "tweet_based_user_video_graph_consumers_based_min_score", default = 4.0, min = 0.0, max = 10.0 ) object MaxConsumerSeedsNumParam extends FSBoundedParam[Int]( name = "tweet_based_user_video_graph_max_user_seeds_num", default = 200, min = 0, max = 500 ) object EnableCoverageExpansionOldTweetParam extends FSParam[Boolean]( name = "tweet_based_user_video_graph_enable_coverage_expansion_old_tweet", default = false ) object EnableCoverageExpansionAllTweetParam extends FSParam[Boolean]( name = "tweet_based_user_video_graph_enable_coverage_expansion_all_tweet", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq( MinCoOccurrenceParam, MaxConsumerSeedsNumParam, TweetBasedMinScoreParam, EnableCoverageExpansionOldTweetParam, EnableCoverageExpansionAllTweetParam ) lazy val config: BaseConfig = { val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( MinCoOccurrenceParam, MaxConsumerSeedsNumParam ) val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(TweetBasedMinScoreParam) BaseConfigBuilder() .set(intOverrides: _*) .set(doubleOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetSharesParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param object TweetSharesParams { object EnableSourceParam extends FSParam[Boolean]( name = "twistly_tweetshares_enable_source", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam, ) BaseConfigBuilder() .set(booleanOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UnifiedSETweetCombinationMethod.scala ================================================ package com.twitter.cr_mixer.param import scala.language.implicitConversions object UnifiedSETweetCombinationMethod extends Enumeration { protected case class CombinationType(s: String) extends super.Val implicit def valueToCombinationType(x: Value): CombinationType = x.asInstanceOf[CombinationType] val Default: Value = CombinationType("") val Interleave: Value = CombinationType("Interleave") val Frontload: Value = CombinationType("Frontload") val Backfill: Value = CombinationType("Backfill") } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UnifiedUSSSignalParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.logging.Logger import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param import com.twitter.usersignalservice.thriftscala.SignalType import scala.language.implicitConversions object UnifiedUSSSignalParams { object TweetAggregationTypeParam extends Enumeration { protected case class SignalTypeValue(signalType: SignalType) extends super.Val implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = x.asInstanceOf[SignalTypeValue] val UniformAggregation = SignalTypeValue(SignalType.TweetBasedUnifiedUniformSignal) val EngagementAggregation = SignalTypeValue( SignalType.TweetBasedUnifiedEngagementWeightedSignal) } object ProducerAggregationTypeParam extends Enumeration { protected case class SignalTypeValue(signalType: SignalType) extends super.Val import scala.language.implicitConversions implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = x.asInstanceOf[SignalTypeValue] val UniformAggregation = SignalTypeValue(SignalType.ProducerBasedUnifiedUniformSignal) val EngagementAggregation = SignalTypeValue( SignalType.ProducerBasedUnifiedEngagementWeightedSignal) } object ReplaceIndividualUSSSourcesParam extends FSParam[Boolean]( name = "twistly_agg_replace_enable_source", default = false ) object EnableTweetAggSourceParam extends FSParam[Boolean]( name = "twistly_agg_tweet_agg_enable_source", default = false ) object TweetAggTypeParam extends FSEnumParam[TweetAggregationTypeParam.type]( name = "twistly_agg_tweet_agg_type_id", default = TweetAggregationTypeParam.EngagementAggregation, enum = TweetAggregationTypeParam ) object UnifiedTweetSourceNumberParam extends FSBoundedParam[Int]( name = "twistly_agg_tweet_agg_source_number", default = 0, min = 0, max = 100, ) object EnableProducerAggSourceParam extends FSParam[Boolean]( name = "twistly_agg_producer_agg_enable_source", default = false ) object ProducerAggTypeParam extends FSEnumParam[ProducerAggregationTypeParam.type]( name = "twistly_agg_producer_agg_type_id", default = ProducerAggregationTypeParam.EngagementAggregation, enum = ProducerAggregationTypeParam ) object UnifiedProducerSourceNumberParam extends FSBoundedParam[Int]( name = "twistly_agg_producer_agg_source_number", default = 0, min = 0, max = 100, ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableTweetAggSourceParam, EnableProducerAggSourceParam, TweetAggTypeParam, ProducerAggTypeParam, UnifiedTweetSourceNumberParam, UnifiedProducerSourceNumberParam, ReplaceIndividualUSSSourcesParam ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableTweetAggSourceParam, EnableProducerAggSourceParam, ReplaceIndividualUSSSourcesParam, ) val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( UnifiedProducerSourceNumberParam, UnifiedTweetSourceNumberParam) val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( NullStatsReceiver, Logger(getClass), TweetAggTypeParam, ProducerAggTypeParam ) BaseConfigBuilder() .set(booleanOverrides: _*) .set(intOverrides: _*) .set(enumOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UtegTweetGlobalParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.conversions.DurationOps._ import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.timelines.configapi.Param import com.twitter.util.Duration object UtegTweetGlobalParams { object MaxUtegCandidatesToRequestParam extends FSBoundedParam[Int]( name = "max_uteg_candidates_to_request", default = 800, min = 10, max = 200 ) object CandidateRefreshSinceTimeOffsetHoursParam extends FSBoundedParam[Duration]( name = "candidate_refresh_since_time_offset_hours", default = 48.hours, min = 1.hours, max = 96.hours ) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromHours } object EnableTLRHealthFilterParam extends FSParam[Boolean]( name = "enable_uteg_tlr_health_filter", default = true ) object EnableRepliesToNonFollowedUsersFilterParam extends FSParam[Boolean]( name = "enable_uteg_replies_to_non_followed_users_filter", default = false ) object EnableRetweetFilterParam extends FSParam[Boolean]( name = "enable_uteg_retweet_filter", default = true ) object EnableInNetworkFilterParam extends FSParam[Boolean]( name = "enable_uteg_in_network_filter", default = true ) val AllParams: Seq[Param[_] with FSName] = Seq( MaxUtegCandidatesToRequestParam, CandidateRefreshSinceTimeOffsetHoursParam, EnableTLRHealthFilterParam, EnableRepliesToNonFollowedUsersFilterParam, EnableRetweetFilterParam, EnableInNetworkFilterParam ) lazy val config: BaseConfig = { val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( MaxUtegCandidatesToRequestParam ) val durationFSOverrides = FeatureSwitchOverrideUtil.getDurationFSOverrides( CandidateRefreshSinceTimeOffsetHoursParam ) val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableTLRHealthFilterParam, EnableRepliesToNonFollowedUsersFilterParam, EnableRetweetFilterParam, EnableInNetworkFilterParam ) BaseConfigBuilder() .set(intOverrides: _*) .set(durationFSOverrides: _*) .set(booleanOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/VideoTweetFilterParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.Param object VideoTweetFilterParams { object EnableVideoTweetFilterParam extends FSParam[Boolean]( name = "video_tweet_filter_enable_filter", default = false ) val AllParams: Seq[Param[_] with FSName] = Seq( EnableVideoTweetFilterParam ) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides(EnableVideoTweetFilterParam) BaseConfigBuilder() .set(booleanOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/VideoViewTweetsParams.scala ================================================ package com.twitter.cr_mixer.param import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.logging.Logger import com.twitter.timelines.configapi.BaseConfig import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import com.twitter.timelines.configapi.Param import com.twitter.usersignalservice.thriftscala.SignalType object VideoViewTweetsParams { object EnableSourceParam extends FSParam[Boolean]( name = "signal_videoviewtweets_enable_source", default = false ) object EnableSourceImpressionParam extends FSParam[Boolean]( name = "signal_videoviewtweets_enableimpression_source", default = false ) object VideoViewTweetType extends Enumeration { protected case class SignalTypeValue(signalType: SignalType) extends super.Val import scala.language.implicitConversions implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = x.asInstanceOf[SignalTypeValue] val VideoTweetQualityView: SignalTypeValue = SignalTypeValue(SignalType.VideoView90dQualityV1) val VideoTweetPlayback50: SignalTypeValue = SignalTypeValue(SignalType.VideoView90dPlayback50V1) } object VideoViewTweetTypeParam extends FSEnumParam[VideoViewTweetType.type]( name = "signal_videoviewtweets_videoviewtype_id", default = VideoViewTweetType.VideoTweetQualityView, enum = VideoViewTweetType ) val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam, EnableSourceImpressionParam, VideoViewTweetTypeParam) lazy val config: BaseConfig = { val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( EnableSourceParam, EnableSourceImpressionParam, ) val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( NullStatsReceiver, Logger(getClass), VideoViewTweetTypeParam) BaseConfigBuilder() .set(booleanOverrides: _*) .set(enumOverrides: _*) .build() } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/javax/inject:javax.inject", "decider/src/main/scala", "finagle/finagle-base-http/src/main", "finagle/finagle-core/src/main", "finagle/finagle-http/src/main/scala", "servo/decider", "src/scala/com/twitter/simclusters_v2/common", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/CrMixerDecider.scala ================================================ package com.twitter.cr_mixer.param.decider import com.twitter.decider.Decider import com.twitter.decider.RandomRecipient import com.twitter.decider.Recipient import com.twitter.decider.SimpleRecipient import com.twitter.simclusters_v2.common.DeciderGateBuilderWithIdHashing import javax.inject.Inject case class CrMixerDecider @Inject() (decider: Decider) { def isAvailable(feature: String, recipient: Option[Recipient]): Boolean = { decider.isAvailable(feature, recipient) } lazy val deciderGateBuilder = new DeciderGateBuilderWithIdHashing(decider) /** * When useRandomRecipient is set to false, the decider is either completely on or off. * When useRandomRecipient is set to true, the decider is on for the specified % of traffic. */ def isAvailable(feature: String, useRandomRecipient: Boolean = true): Boolean = { if (useRandomRecipient) isAvailable(feature, Some(RandomRecipient)) else isAvailable(feature, None) } /*** * Decide whether the decider is available for a specific id using SimpleRecipient(id). */ def isAvailableForId( id: Long, deciderConstants: String ): Boolean = { // Note: SimpleRecipient does expose a `val isUser = true` field which is not correct if the Id is not a user Id. // However this field does not appear to be used anywhere in source. decider.isAvailable(deciderConstants, Some(SimpleRecipient(id))) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/DeciderKey.scala ================================================ package com.twitter.cr_mixer.param.decider import com.twitter.servo.decider.DeciderKeyEnum object DeciderConstants { val enableHealthSignalsScoreDeciderKey = "enable_tweet_health_score" val enableUTGRealTimeTweetEngagementScoreDeciderKey = "enable_utg_realtime_tweet_engagement_score" val enableUserAgathaScoreDeciderKey = "enable_user_agatha_score" val enableUserTweetEntityGraphTrafficDeciderKey = "enable_user_tweet_entity_graph_traffic" val enableUserTweetGraphTrafficDeciderKey = "enable_user_tweet_graph_traffic" val enableUserVideoGraphTrafficDeciderKey = "enable_user_video_graph_traffic" val enableUserAdGraphTrafficDeciderKey = "enable_user_ad_graph_traffic" val enableSimClustersANN2DarkTrafficDeciderKey = "enable_simclusters_ann_2_dark_traffic" val enableQigSimilarTweetsTrafficDeciderKey = "enable_qig_similar_tweets_traffic" val enableFRSTrafficDeciderKey = "enable_frs_traffic" val upperFunnelPerStepScribeRate = "upper_funnel_per_step_scribe_rate" val kafkaMessageScribeSampleRate = "kafka_message_scribe_sample_rate" val enableRealGraphMhStoreDeciderKey = "enable_real_graph_mh_store" val topLevelApiDdgMetricsScribeRate = "top_level_api_ddg_metrics_scribe_rate" val adsRecommendationsPerExperimentScribeRate = "ads_recommendations_per_experiment_scribe_rate" val enableScribeForBlueVerifiedTweetCandidates = "enable_scribe_for_blue_verified_tweet_candidates" val enableUserStateStoreDeciderKey = "enable_user_state_store" val enableUserMediaRepresentationStoreDeciderKey = "enable_user_media_representation_store" val enableMagicRecsRealTimeAggregatesStoreDeciderKey = "enable_magic_recs_real_time_aggregates_store" val enableEarlybirdTrafficDeciderKey = "enable_earlybird_traffic" val enableTopicTweetTrafficDeciderKey = "enable_topic_tweet_traffic" val getTweetRecommendationsCacheRate = "get_tweet_recommendations_cache_rate" } object DeciderKey extends DeciderKeyEnum { val enableHealthSignalsScoreDeciderKey: Value = Value( DeciderConstants.enableHealthSignalsScoreDeciderKey ) val enableUtgRealTimeTweetEngagementScoreDeciderKey: Value = Value( DeciderConstants.enableUTGRealTimeTweetEngagementScoreDeciderKey ) val enableUserAgathaScoreDeciderKey: Value = Value( DeciderConstants.enableUserAgathaScoreDeciderKey ) val enableUserMediaRepresentationStoreDeciderKey: Value = Value( DeciderConstants.enableUserMediaRepresentationStoreDeciderKey ) val enableMagicRecsRealTimeAggregatesStore: Value = Value( DeciderConstants.enableMagicRecsRealTimeAggregatesStoreDeciderKey ) val enableUserStateStoreDeciderKey: Value = Value( DeciderConstants.enableUserStateStoreDeciderKey ) val enableRealGraphMhStoreDeciderKey: Value = Value( DeciderConstants.enableRealGraphMhStoreDeciderKey ) val enableEarlybirdTrafficDeciderKey: Value = Value( DeciderConstants.enableEarlybirdTrafficDeciderKey) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/EndpointLoadShedder.scala ================================================ package com.twitter.cr_mixer.param.decider import com.twitter.decider.Decider import com.twitter.decider.RandomRecipient import com.twitter.finagle.stats.StatsReceiver import com.twitter.util.Future import javax.inject.Inject import scala.util.control.NoStackTrace /* Provides deciders-controlled load shedding for a given Product from a given endpoint. The format of the decider keys is: enable_loadshedding__ E.g.: enable_loadshedding_getTweetRecommendations_Notifications Deciders are fractional, so a value of 50.00 will drop 50% of responses. If a decider key is not defined for a particular endpoint/product combination, those requests will always be served. We should therefore aim to define keys for the endpoints/product we care most about in decider.yml, so that we can control them during incidents. */ case class EndpointLoadShedder @Inject() ( decider: Decider, statsReceiver: StatsReceiver) { import EndpointLoadShedder._ // Fall back to False for any undefined key private val deciderWithFalseFallback: Decider = decider.orElse(Decider.False) private val keyPrefix = "enable_loadshedding" private val scopedStats = statsReceiver.scope("EndpointLoadShedder") def apply[T](endpointName: String, product: String)(serve: => Future[T]): Future[T] = { /* Checks if either per-product or top-level load shedding is enabled If both are enabled at different percentages, load shedding will not be perfectly calculable due to salting of hash (i.e. 25% load shed for Product x + 25% load shed for overall does not result in 50% load shed for x) */ val keyTyped = s"${keyPrefix}_${endpointName}_$product" val keyTopLevel = s"${keyPrefix}_${endpointName}" if (deciderWithFalseFallback.isAvailable(keyTopLevel, recipient = Some(RandomRecipient))) { scopedStats.counter(keyTopLevel).incr Future.exception(LoadSheddingException) } else if (deciderWithFalseFallback.isAvailable(keyTyped, recipient = Some(RandomRecipient))) { scopedStats.counter(keyTyped).incr Future.exception(LoadSheddingException) } else serve } } object EndpointLoadShedder { object LoadSheddingException extends Exception with NoStackTrace } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/storehaus:core", "3rdparty/jvm/javax/inject:javax.inject", "configapi/configapi-core", "content-recommender/thrift/src/main/thrift:content-recommender-common-scala", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/config", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util", "cr-mixer/thrift/src/main/thrift:thrift-scala", "decider/src/main/scala", "frigate/frigate-common:base", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/base", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util:stats_util", "hydra/common/libraries/src/main/scala/com/twitter/hydra/common/model_config", "hydra/partition/thrift/src/main/thrift:thrift-scala", "hydra/root/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", "src/scala/com/twitter/simclusters_v2/common", "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", "src/thrift/com/twitter/ml/api:data-scala", "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/DefaultRanker.scala ================================================ package com.twitter.cr_mixer.ranker import com.twitter.cr_mixer.model.BlendedCandidate import com.twitter.cr_mixer.model.RankedCandidate import com.twitter.util.Future import javax.inject.Singleton /** * Keep the same order as the input. */ @Singleton class DefaultRanker() { def rank( candidates: Seq[BlendedCandidate], ): Future[Seq[RankedCandidate]] = { val candidateSize = candidates.size val rankedCandidates = candidates.zipWithIndex.map { case (candidate, index) => candidate.toRankedCandidate((candidateSize - index).toDouble) } Future.value(rankedCandidates) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/SwitchRanker.scala ================================================ package com.twitter.cr_mixer.ranker import com.twitter.cr_mixer.model.BlendedCandidate import com.twitter.cr_mixer.model.CrCandidateGeneratorQuery import com.twitter.cr_mixer.model.RankedCandidate import com.twitter.finagle.stats.StatsReceiver import com.twitter.util.Future import com.twitter.util.JavaTimer import com.twitter.util.Time import com.twitter.util.Timer import javax.inject.Inject import javax.inject.Singleton /** * CR-Mixer internal ranker */ @Singleton class SwitchRanker @Inject() ( defaultRanker: DefaultRanker, globalStats: StatsReceiver) { private val stats: StatsReceiver = globalStats.scope(this.getClass.getCanonicalName) implicit val timer: Timer = new JavaTimer(true) def rank( query: CrCandidateGeneratorQuery, candidates: Seq[BlendedCandidate], ): Future[Seq[RankedCandidate]] = { defaultRanker.rank(candidates) } } object SwitchRanker { /** Prefers candidates generated from sources with the latest timestamps. * The newer the source signal, the higher a candidate ranks. * This ordering biases against consumer-based candidates because their timestamp defaults to 0 */ val TimestampOrder: Ordering[RankedCandidate] = math.Ordering .by[RankedCandidate, Time]( _.reasonChosen.sourceInfoOpt .flatMap(_.sourceEventTime) .getOrElse(Time.fromMilliseconds(0L))) .reverse } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/javax/inject:javax.inject", "configapi/configapi-core", "cr-mixer/thrift/src/main/thrift:thrift-scala", "decider/src/main/scala", "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", "finagle/finagle-core/src/main", "frigate/frigate-common:base", "frigate/frigate-common:util", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/base", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", "scrooge/scrooge-serializer/src/main/scala", "src/scala/com/twitter/simclusters_v2/common", "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", "util-internal/scribe/src/main/scala/com/twitter/logging", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe/ScribeCategory.scala ================================================ package com.twitter.cr_mixer.scribe /** * Categories define scribe categories used in cr-mixer service. */ object ScribeCategories { lazy val AllCategories = List(AbDecider, TopLevelApiDdgMetrics, TweetsRecs) /** * AbDecider represents scribe logs for experiments */ lazy val AbDecider: ScribeCategory = ScribeCategory( "abdecider_scribe", "client_event" ) /** * Top-Level Client event scribe logs, to record changes in system metrics (e.g. latency, * candidates returned, empty rate ) per experiment bucket, and store them in DDG metric group */ lazy val TopLevelApiDdgMetrics: ScribeCategory = ScribeCategory( "top_level_api_ddg_metrics_scribe", "client_event" ) lazy val TweetsRecs: ScribeCategory = ScribeCategory( "get_tweets_recommendations_scribe", "cr_mixer_get_tweets_recommendations" ) lazy val VITTweetsRecs: ScribeCategory = ScribeCategory( "get_vit_tweets_recommendations_scribe", "cr_mixer_get_vit_tweets_recommendations" ) lazy val RelatedTweets: ScribeCategory = ScribeCategory( "get_related_tweets_scribe", "cr_mixer_get_related_tweets" ) lazy val UtegTweets: ScribeCategory = ScribeCategory( "get_uteg_tweets_scribe", "cr_mixer_get_uteg_tweets" ) lazy val AdsRecommendations: ScribeCategory = ScribeCategory( "get_ads_recommendations_scribe", "cr_mixer_get_ads_recommendations" ) } /** * Category represents each scribe log data. * * @param loggerFactoryNode loggerFactory node name in cr-mixer associated with this scribe category * @param scribeCategory scribe category name (globally unique at Twitter) */ case class ScribeCategory( loggerFactoryNode: String, scribeCategory: String) { def getProdLoggerFactoryNode: String = loggerFactoryNode def getStagingLoggerFactoryNode: String = "staging_" + loggerFactoryNode } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/service/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/javax/inject:javax.inject", "configapi/configapi-core", "cr-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", "stitch/stitch-core", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/service/CrMixerAlertNotificationConfig.scala ================================================ package com.twitter.cr_mixer.service import com.twitter.product_mixer.core.functional_component.common.alert.Destination import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup /** * Notifications (email, pagerduty, etc) can be specific per-alert but it is common for multiple * products to share notification configuration. * * Our configuration uses only email notifications because SampleMixer is a demonstration service * with neither internal nor customer-facing users. You will likely want to use a PagerDuty * destination instead. For example: * {{{ * critical = Destination(pagerDutyKey = Some("your-pagerduty-key")) * }}} * * * For more information about how to get a PagerDuty key, see: * https://docbird.twitter.biz/mon/how-to-guides.html?highlight=notificationgroup#set-up-email-pagerduty-and-slack-notifications */ object CrMixerAlertNotificationConfig { val DefaultNotificationGroup: NotificationGroup = NotificationGroup( warn = Destination(emails = Seq("no-reply@twitter.com")), critical = Destination(emails = Seq("no-reply@twitter.com")) ) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/BUILD ================================================ scala_library( sources = [ "*.scala", ], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/storehaus:core", "3rdparty/jvm/com/twitter/storehaus:memcache", "3rdparty/jvm/io/grpc:grpc-api", "3rdparty/jvm/io/grpc:grpc-auth", "3rdparty/jvm/io/grpc:grpc-core", "3rdparty/jvm/io/grpc:grpc-netty", "3rdparty/jvm/io/grpc:grpc-protobuf", "3rdparty/jvm/io/grpc:grpc-stub", "3rdparty/jvm/io/opil:tensorflow-serving-client", "3rdparty/jvm/javax/inject:javax.inject", "3rdparty/src/jvm/com/twitter/storehaus:core", "ann/src/main/scala/com/twitter/ann/hnsw", "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", "configapi/configapi-core", "content-recommender/thrift/src/main/thrift:thrift-scala", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/config", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/exception", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util", "cr-mixer/thrift/src/main/thrift:thrift-scala", "decider/src/main/scala", "finagle-internal/finagle-grpc/src/main/scala", "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/client", "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "frigate/frigate-common:base", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/base", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/candidate", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util:stats_util", "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", "mediaservices/commons/src/main/scala:futuretracker", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", "qig-ranker/thrift/src/main/thrift:thrift-scala", "relevance-platform/src/main/scala/com/twitter/relevance_platform/common/injection", "relevance-platform/src/main/scala/com/twitter/relevance_platform/simclustersann/multicluster", "simclusters-ann/thrift/src/main/thrift:thrift-scala", "snowflake/src/main/scala/com/twitter/snowflake/id", "src/java/com/twitter/search/common/schema/base", "src/java/com/twitter/search/common/schema/earlybird", "src/java/com/twitter/search/queryparser/query:core-query-nodes", "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", "src/scala/com/twitter/cortex/ml/embeddings/common:Helpers", "src/scala/com/twitter/ml/featurestore/lib", "src/scala/com/twitter/simclusters_v2/common", "src/thrift/com/twitter/ml/api:embedding-scala", "src/thrift/com/twitter/recos:recos-common-scala", "src/thrift/com/twitter/recos/user_ad_graph:user_ad_graph-scala", "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", "src/thrift/com/twitter/recos/user_tweet_graph:user_tweet_graph-scala", "src/thrift/com/twitter/recos/user_video_graph:user_video_graph-scala", "src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/search/common:ranking-scala", "src/thrift/com/twitter/search/query_interaction_graph/service:qig-service-scala", "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", "src/thrift/com/twitter/topic_recos:topic_recos-thrift-scala", "src/thrift/com/twitter/trends/trip_v1:trip-tweets-thrift-scala", "src/thrift/com/twitter/twistly:twistly-scala", "strato/src/main/scala/com/twitter/strato/client", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/CertoTopicTweetSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.google.inject.Inject import com.google.inject.Singleton import com.google.inject.name.Named import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TopicTweetWithScore import com.twitter.cr_mixer.param.TopicTweetParams import com.twitter.cr_mixer.similarity_engine.CertoTopicTweetSimilarityEngine._ import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.simclusters_v2.thriftscala.TopicId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.topic_recos.thriftscala._ import com.twitter.util.Future @Singleton case class CertoTopicTweetSimilarityEngine @Inject() ( @Named(ModuleNames.CertoStratoStoreName) certoStratoStore: ReadableStore[ TopicId, Seq[TweetWithScores] ], statsReceiver: StatsReceiver) extends ReadableStore[EngineQuery[Query], Seq[TopicTweetWithScore]] { private val name: String = this.getClass.getSimpleName private val stats = statsReceiver.scope(name) override def get(query: EngineQuery[Query]): Future[Option[Seq[TopicTweetWithScore]]] = { StatsUtil.trackOptionItemsStats(stats) { topTweetsByFollowerL2NormalizedScore.get(query).map { _.map { topicTopTweets => topicTopTweets.map { topicTweet => TopicTweetWithScore( tweetId = topicTweet.tweetId, score = topicTweet.scores.followerL2NormalizedCosineSimilarity8HrHalfLife, similarityEngineType = SimilarityEngineType.CertoTopicTweet ) } } } } } private val topTweetsByFollowerL2NormalizedScore: ReadableStore[EngineQuery[Query], Seq[ TweetWithScores ]] = { ReadableStore.fromFnFuture { query: EngineQuery[Query] => StatsUtil.trackOptionItemsStats(stats) { for { topKTweetsWithScores <- certoStratoStore.get(query.storeQuery.topicId) } yield { topKTweetsWithScores.map( _.filter( _.scores.followerL2NormalizedCosineSimilarity8HrHalfLife >= query.storeQuery.certoScoreTheshold) .take(query.storeQuery.maxCandidates)) } } } } } object CertoTopicTweetSimilarityEngine { // Query is used as a cache key. Do not add any user level information in this. case class Query( topicId: TopicId, maxCandidates: Int, certoScoreTheshold: Double) def fromParams( topicId: TopicId, isVideoOnly: Boolean, params: configapi.Params, ): EngineQuery[Query] = { val maxCandidates = if (isVideoOnly) { params(TopicTweetParams.MaxCertoCandidatesParam) * 2 } else { params(TopicTweetParams.MaxCertoCandidatesParam) } EngineQuery( Query( topicId = topicId, maxCandidates = maxCandidates, certoScoreTheshold = params(TopicTweetParams.CertoScoreThresholdParam) ), params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerBasedWalsSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.ConsumerBasedWalsParams import com.twitter.cr_mixer.similarity_engine.ConsumerBasedWalsSimilarityEngine.Query import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.finagle.stats.StatsReceiver import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.util.Future import io.grpc.ManagedChannel import tensorflow.serving.Predict.PredictRequest import tensorflow.serving.Predict.PredictResponse import tensorflow.serving.PredictionServiceGrpc import org.tensorflow.example.Feature import org.tensorflow.example.Int64List import org.tensorflow.example.FloatList import org.tensorflow.example.Features import org.tensorflow.example.Example import tensorflow.serving.Model import org.tensorflow.framework.TensorProto import org.tensorflow.framework.DataType import org.tensorflow.framework.TensorShapeProto import com.twitter.finagle.grpc.FutureConverters import java.util.ArrayList import java.lang import com.twitter.util.Return import com.twitter.util.Throw import java.util.concurrent.ConcurrentHashMap import scala.jdk.CollectionConverters._ // Stats object maintain a set of stats that are specific to the Wals Engine. case class WalsStats(scope: String, scopedStats: StatsReceiver) { val requestStat = scopedStats.scope(scope) val inputSignalSize = requestStat.stat("input_signal_size") val latency = requestStat.stat("latency_ms") val latencyOnError = requestStat.stat("error_latency_ms") val latencyOnSuccess = requestStat.stat("success_latency_ms") val requests = requestStat.counter("requests") val success = requestStat.counter("success") val failures = requestStat.scope("failures") def onFailure(t: Throwable, startTimeMs: Long) { val duration = System.currentTimeMillis() - startTimeMs latency.add(duration) latencyOnError.add(duration) failures.counter(t.getClass.getName).incr() } def onSuccess(startTimeMs: Long) { val duration = System.currentTimeMillis() - startTimeMs latency.add(duration) latencyOnSuccess.add(duration) success.incr() } } // StatsMap maintains a mapping from Model's input signature to a stats receiver // The Wals model suports multiple input signature which can run different graphs internally and // can have a different performance profile. // Invoking StatsReceiver.stat() on each request can create a new stat object and can be expensive // in performance critical paths. object WalsStatsMap { val mapping = new ConcurrentHashMap[String, WalsStats]() def get(scope: String, scopedStats: StatsReceiver): WalsStats = { mapping.computeIfAbsent(scope, (scope) => WalsStats(scope, scopedStats)) } } case class ConsumerBasedWalsSimilarityEngine( homeNaviGRPCClient: ManagedChannel, adsFavedNaviGRPCClient: ManagedChannel, adsMonetizableNaviGRPCClient: ManagedChannel, statsReceiver: StatsReceiver) extends ReadableStore[ Query, Seq[TweetWithScore] ] { override def get( query: ConsumerBasedWalsSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { val startTimeMs = System.currentTimeMillis() val stats = WalsStatsMap.get( query.wilyNsName + "/" + query.modelSignatureName, statsReceiver.scope("NaviPredictionService") ) stats.requests.incr() stats.inputSignalSize.add(query.sourceIds.size) try { // avoid inference calls is source signals are empty if (query.sourceIds.isEmpty) { Future.value(Some(Seq.empty)) } else { val grpcClient = query.wilyNsName match { case "navi-wals-recommended-tweets-home-client" => homeNaviGRPCClient case "navi-wals-ads-faved-tweets" => adsFavedNaviGRPCClient case "navi-wals-ads-monetizable-tweets" => adsFavedNaviGRPCClient // default to homeNaviGRPCClient case _ => homeNaviGRPCClient } val stub = PredictionServiceGrpc.newFutureStub(grpcClient) val inferRequest = getModelInput(query) FutureConverters .RichListenableFuture(stub.predict(inferRequest)).toTwitter .transform { case Return(resp) => stats.onSuccess(startTimeMs) Future.value(Some(getModelOutput(query, resp))) case Throw(e) => stats.onFailure(e, startTimeMs) Future.exception(e) } } } catch { case e: Throwable => Future.exception(e) } } def getFeaturesForRecommendations(query: ConsumerBasedWalsSimilarityEngine.Query): Example = { val tweetIds = new ArrayList[lang.Long]() val tweetFaveWeight = new ArrayList[lang.Float]() query.sourceIds.foreach { sourceInfo => val weight = sourceInfo.sourceType match { case SourceType.TweetFavorite | SourceType.Retweet => 1.0f // currently no-op - as we do not get negative signals case SourceType.TweetDontLike | SourceType.TweetReport | SourceType.AccountMute | SourceType.AccountBlock => 0.0f case _ => 0.0f } sourceInfo.internalId match { case InternalId.TweetId(tweetId) => tweetIds.add(tweetId) tweetFaveWeight.add(weight) case _ => throw new IllegalArgumentException( s"Invalid InternalID - does not contain TweetId for Source Signal: ${sourceInfo}") } } val tweetIdsFeature = Feature .newBuilder().setInt64List( Int64List .newBuilder().addAllValue(tweetIds).build() ).build() val tweetWeightsFeature = Feature .newBuilder().setFloatList( FloatList.newBuilder().addAllValue(tweetFaveWeight).build()).build() val features = Features .newBuilder() .putFeature("tweet_ids", tweetIdsFeature) .putFeature("tweet_weights", tweetWeightsFeature) .build() Example.newBuilder().setFeatures(features).build() } def getModelInput(query: ConsumerBasedWalsSimilarityEngine.Query): PredictRequest = { val tfExample = getFeaturesForRecommendations(query) val inferenceRequest = PredictRequest .newBuilder() .setModelSpec( Model.ModelSpec .newBuilder() .setName(query.modelName) .setSignatureName(query.modelSignatureName)) .putInputs( query.modelInputName, TensorProto .newBuilder() .setDtype(DataType.DT_STRING) .setTensorShape(TensorShapeProto .newBuilder() .addDim(TensorShapeProto.Dim.newBuilder().setSize(1))) .addStringVal(tfExample.toByteString) .build() ).build() inferenceRequest } def getModelOutput(query: Query, response: PredictResponse): Seq[TweetWithScore] = { val outputName = query.modelOutputName if (response.containsOutputs(outputName)) { val tweetList = response.getOutputsMap .get(outputName) .getInt64ValList.asScala tweetList.zip(tweetList.size to 1 by -1).map { (tweetWithScore) => TweetWithScore(tweetWithScore._1, tweetWithScore._2.toLong) } } else { Seq.empty } } } object ConsumerBasedWalsSimilarityEngine { case class Query( sourceIds: Seq[SourceInfo], modelName: String, modelInputName: String, modelOutputName: String, modelSignatureName: String, wilyNsName: String, ) def fromParams( sourceIds: Seq[SourceInfo], params: configapi.Params, ): EngineQuery[Query] = { EngineQuery( Query( sourceIds, params(ConsumerBasedWalsParams.ModelNameParam), params(ConsumerBasedWalsParams.ModelInputNameParam), params(ConsumerBasedWalsParams.ModelOutputNameParam), params(ConsumerBasedWalsParams.ModelSignatureNameParam), params(ConsumerBasedWalsParams.WilyNsNameParam), ), params ) } def toSimilarityEngineInfo( score: Double ): SimilarityEngineInfo = { SimilarityEngineInfo( similarityEngineType = SimilarityEngineType.ConsumerBasedWalsANN, modelId = None, score = Some(score)) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTripSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.TripTweetWithScore import com.twitter.cr_mixer.param.ConsumerEmbeddingBasedTripParams import com.twitter.cr_mixer.util.InterleaveUtil import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.simclusters_v2.common.ClusterId import com.twitter.simclusters_v2.common.SimClustersEmbedding import com.twitter.simclusters_v2.common.UserId import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.timelines.configapi.Params import com.twitter.trends.trip_v1.trip_tweets.thriftscala.Cluster import com.twitter.trends.trip_v1.trip_tweets.thriftscala.ClusterDomain import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweet import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain import com.twitter.util.Future case class TripEngineQuery( modelId: String, sourceId: InternalId, tripSourceId: String, maxResult: Int, params: Params) case class ConsumerEmbeddingBasedTripSimilarityEngine( embeddingStoreLookUpMap: Map[String, ReadableStore[UserId, SimClustersEmbedding]], tripCandidateSource: ReadableStore[TripDomain, Seq[TripTweet]], statsReceiver: StatsReceiver, ) extends ReadableStore[TripEngineQuery, Seq[TripTweetWithScore]] { import ConsumerEmbeddingBasedTripSimilarityEngine._ private val scopedStats = statsReceiver.scope(name) private def fetchTopClusters(query: TripEngineQuery): Future[Option[Seq[ClusterId]]] = { query.sourceId match { case InternalId.UserId(userId) => val embeddingStore = embeddingStoreLookUpMap.getOrElse( query.modelId, throw new IllegalArgumentException( s"${this.getClass.getSimpleName}: " + s"ModelId ${query.modelId} does not exist for embeddingStore" ) ) embeddingStore.get(userId).map(_.map(_.topClusterIds(MaxClusters))) case _ => Future.None } } private def fetchCandidates( topClusters: Seq[ClusterId], tripSourceId: String ): Future[Seq[Seq[TripTweetWithScore]]] = { Future .collect { topClusters.map { clusterId => tripCandidateSource .get( TripDomain( sourceId = tripSourceId, clusterDomain = Some( ClusterDomain(simCluster = Some(Cluster(clusterIntId = Some(clusterId))))))).map { _.map { _.collect { case TripTweet(tweetId, score) => TripTweetWithScore(tweetId, score) } }.getOrElse(Seq.empty).take(MaxNumResultsPerCluster) } } } } override def get(engineQuery: TripEngineQuery): Future[Option[Seq[TripTweetWithScore]]] = { val fetchTopClustersStat = scopedStats.scope(engineQuery.modelId).scope("fetchTopClusters") val fetchCandidatesStat = scopedStats.scope(engineQuery.modelId).scope("fetchCandidates") for { topClustersOpt <- StatsUtil.trackOptionStats(fetchTopClustersStat) { fetchTopClusters(engineQuery) } candidates <- StatsUtil.trackItemsStats(fetchCandidatesStat) { topClustersOpt match { case Some(topClusters) => fetchCandidates(topClusters, engineQuery.tripSourceId) case None => Future.Nil } } } yield { val interleavedTweets = InterleaveUtil.interleave(candidates) val dedupCandidates = interleavedTweets .groupBy(_.tweetId).flatMap { case (_, tweetWithScoreSeq) => tweetWithScoreSeq.sortBy(-_.score).take(1) }.toSeq.take(engineQuery.maxResult) Some(dedupCandidates) } } } object ConsumerEmbeddingBasedTripSimilarityEngine { private val MaxClusters: Int = 8 private val MaxNumResultsPerCluster: Int = 25 private val name: String = this.getClass.getSimpleName def fromParams( modelId: String, sourceId: InternalId, params: configapi.Params ): TripEngineQuery = { TripEngineQuery( modelId = modelId, sourceId = sourceId, tripSourceId = params(ConsumerEmbeddingBasedTripParams.SourceIdParam), maxResult = params(ConsumerEmbeddingBasedTripParams.MaxNumCandidatesParam), params = params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTwHINSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.param.ConsumerEmbeddingBasedTwHINParams import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.timelines.configapi object ConsumerEmbeddingBasedTwHINSimilarityEngine { def fromParams( sourceId: InternalId, params: configapi.Params, ): HnswANNEngineQuery = { HnswANNEngineQuery( sourceId = sourceId, modelId = params(ConsumerEmbeddingBasedTwHINParams.ModelIdParam), params = params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTwoTowerSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.param.ConsumerEmbeddingBasedTwoTowerParams import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.timelines.configapi object ConsumerEmbeddingBasedTwoTowerSimilarityEngine { def fromParams( sourceId: InternalId, params: configapi.Params, ): HnswANNEngineQuery = { HnswANNEngineQuery( sourceId = sourceId, modelId = params(ConsumerEmbeddingBasedTwoTowerParams.ModelIdParam), params = params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumersBasedUserAdGraphSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.ConsumersBasedUserAdGraphParams import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.recos.user_ad_graph.thriftscala.ConsumersBasedRelatedAdRequest import com.twitter.recos.user_ad_graph.thriftscala.RelatedAdResponse import com.twitter.simclusters_v2.common.UserId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.util.Future import javax.inject.Singleton /** * This store uses the graph based input (a list of userIds) * to query consumersBasedUserAdGraph and get their top engaged ad tweets */ @Singleton case class ConsumersBasedUserAdGraphSimilarityEngine( consumersBasedUserAdGraphStore: ReadableStore[ ConsumersBasedRelatedAdRequest, RelatedAdResponse ], statsReceiver: StatsReceiver) extends ReadableStore[ ConsumersBasedUserAdGraphSimilarityEngine.Query, Seq[TweetWithScore] ] { override def get( query: ConsumersBasedUserAdGraphSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { val consumersBasedRelatedAdRequest = ConsumersBasedRelatedAdRequest( query.seedWithScores.keySet.toSeq, maxResults = Some(query.maxResults), minCooccurrence = Some(query.minCooccurrence), minScore = Some(query.minScore), maxTweetAgeInHours = Some(query.maxTweetAgeInHours) ) consumersBasedUserAdGraphStore .get(consumersBasedRelatedAdRequest) .map { relatedAdResponseOpt => relatedAdResponseOpt.map { relatedAdResponse => relatedAdResponse.adTweets.map { tweet => TweetWithScore(tweet.adTweetId, tweet.score) } } } } } object ConsumersBasedUserAdGraphSimilarityEngine { case class Query( seedWithScores: Map[UserId, Double], maxResults: Int, minCooccurrence: Int, minScore: Double, maxTweetAgeInHours: Int) def toSimilarityEngineInfo( score: Double ): SimilarityEngineInfo = { SimilarityEngineInfo( similarityEngineType = SimilarityEngineType.ConsumersBasedUserAdGraph, modelId = None, score = Some(score)) } def fromParams( seedWithScores: Map[UserId, Double], params: configapi.Params, ): EngineQuery[Query] = { EngineQuery( Query( seedWithScores = seedWithScores, maxResults = params(GlobalParams.MaxCandidateNumPerSourceKeyParam), minCooccurrence = params(ConsumersBasedUserAdGraphParams.MinCoOccurrenceParam), minScore = params(ConsumersBasedUserAdGraphParams.MinScoreParam), maxTweetAgeInHours = params(GlobalParams.MaxTweetAgeHoursParam).inHours, ), params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumersBasedUserVideoGraphSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.ConsumersBasedUserVideoGraphParams import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.recos.user_video_graph.thriftscala.ConsumersBasedRelatedTweetRequest import com.twitter.recos.user_video_graph.thriftscala.RelatedTweetResponse import com.twitter.simclusters_v2.common.UserId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.util.Future import javax.inject.Singleton /** * This store uses the graph based input (a list of userIds) * to query consumersBasedUserVideoGraph and get their top engaged tweets */ @Singleton case class ConsumersBasedUserVideoGraphSimilarityEngine( consumersBasedUserVideoGraphStore: ReadableStore[ ConsumersBasedRelatedTweetRequest, RelatedTweetResponse ], statsReceiver: StatsReceiver) extends ReadableStore[ ConsumersBasedUserVideoGraphSimilarityEngine.Query, Seq[TweetWithScore] ] { override def get( query: ConsumersBasedUserVideoGraphSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { val consumersBasedRelatedTweetRequest = ConsumersBasedRelatedTweetRequest( query.seedWithScores.keySet.toSeq, maxResults = Some(query.maxResults), minCooccurrence = Some(query.minCooccurrence), minScore = Some(query.minScore), maxTweetAgeInHours = Some(query.maxTweetAgeInHours) ) consumersBasedUserVideoGraphStore .get(consumersBasedRelatedTweetRequest) .map { relatedTweetResponseOpt => relatedTweetResponseOpt.map { relatedTweetResponse => relatedTweetResponse.tweets.map { tweet => TweetWithScore(tweet.tweetId, tweet.score) } } } } } object ConsumersBasedUserVideoGraphSimilarityEngine { case class Query( seedWithScores: Map[UserId, Double], maxResults: Int, minCooccurrence: Int, minScore: Double, maxTweetAgeInHours: Int) def toSimilarityEngineInfo( score: Double ): SimilarityEngineInfo = { SimilarityEngineInfo( similarityEngineType = SimilarityEngineType.ConsumersBasedUserVideoGraph, modelId = None, score = Some(score)) } def fromParamsForRealGraphIn( seedWithScores: Map[UserId, Double], params: configapi.Params, ): EngineQuery[Query] = { EngineQuery( Query( seedWithScores = seedWithScores, maxResults = params(GlobalParams.MaxCandidateNumPerSourceKeyParam), minCooccurrence = params(ConsumersBasedUserVideoGraphParams.RealGraphInMinCoOccurrenceParam), minScore = params(ConsumersBasedUserVideoGraphParams.RealGraphInMinScoreParam), maxTweetAgeInHours = params(GlobalParams.MaxTweetAgeHoursParam).inHours ), params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/DiffusionBasedSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.simclusters_v2.thriftscala.TweetsWithScore import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.util.Future import javax.inject.Singleton @Singleton case class DiffusionBasedSimilarityEngine( retweetBasedDiffusionRecsMhStore: ReadableStore[Long, TweetsWithScore], statsReceiver: StatsReceiver) extends ReadableStore[ DiffusionBasedSimilarityEngine.Query, Seq[TweetWithScore] ] { override def get( query: DiffusionBasedSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { query.sourceId match { case InternalId.UserId(userId) => retweetBasedDiffusionRecsMhStore.get(userId).map { _.map { tweetsWithScore => { tweetsWithScore.tweets .map(tweet => TweetWithScore(tweet.tweetId, tweet.score)) } } } case _ => Future.None } } } object DiffusionBasedSimilarityEngine { val defaultScore: Double = 0.0 case class Query( sourceId: InternalId, ) def toSimilarityEngineInfo( query: LookupEngineQuery[Query], score: Double ): SimilarityEngineInfo = { SimilarityEngineInfo( similarityEngineType = SimilarityEngineType.DiffusionBasedTweet, modelId = Some(query.lookupKey), score = Some(score)) } def fromParams( sourceId: InternalId, modelId: String, params: configapi.Params, ): LookupEngineQuery[Query] = { LookupEngineQuery( Query(sourceId = sourceId), modelId, params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdModelBasedSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.similarity_engine.EarlybirdModelBasedSimilarityEngine.EarlybirdModelBasedSearchQuery import com.twitter.cr_mixer.similarity_engine.EarlybirdSimilarityEngineBase._ import com.twitter.cr_mixer.util.EarlybirdSearchUtil.EarlybirdClientId import com.twitter.cr_mixer.util.EarlybirdSearchUtil.FacetsToFetch import com.twitter.cr_mixer.util.EarlybirdSearchUtil.MetadataOptions import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.tracing.Trace import com.twitter.search.common.ranking.thriftscala.ThriftRankingParams import com.twitter.search.common.ranking.thriftscala.ThriftScoringFunctionType import com.twitter.search.earlybird.thriftscala.EarlybirdRequest import com.twitter.search.earlybird.thriftscala.EarlybirdService import com.twitter.search.earlybird.thriftscala.ThriftSearchQuery import com.twitter.search.earlybird.thriftscala.ThriftSearchRankingMode import com.twitter.search.earlybird.thriftscala.ThriftSearchRelevanceOptions import com.twitter.simclusters_v2.common.UserId import javax.inject.Inject import javax.inject.Singleton @Singleton case class EarlybirdModelBasedSimilarityEngine @Inject() ( earlybirdSearchClient: EarlybirdService.MethodPerEndpoint, timeoutConfig: TimeoutConfig, stats: StatsReceiver) extends EarlybirdSimilarityEngineBase[EarlybirdModelBasedSearchQuery] { import EarlybirdModelBasedSimilarityEngine._ override val statsReceiver: StatsReceiver = stats.scope(this.getClass.getSimpleName) override def getEarlybirdRequest( query: EarlybirdModelBasedSearchQuery ): Option[EarlybirdRequest] = if (query.seedUserIds.nonEmpty) Some( EarlybirdRequest( searchQuery = getThriftSearchQuery(query), clientId = Some(EarlybirdClientId), timeoutMs = timeoutConfig.earlybirdServerTimeout.inMilliseconds.intValue(), clientRequestID = Some(s"${Trace.id.traceId}"), )) else None } object EarlybirdModelBasedSimilarityEngine { case class EarlybirdModelBasedSearchQuery( seedUserIds: Seq[UserId], maxNumTweets: Int, oldestTweetTimestampInSec: Option[UserId], frsUserToScoresForScoreAdjustment: Option[Map[UserId, Double]]) extends EarlybirdSearchQuery /** * Used by Push Service */ val RealGraphScoringModel = "frigate_unified_engagement_rg" val MaxHitsToProcess = 1000 val MaxConsecutiveSameUser = 1 private def getModelBasedRankingParams( authorSpecificScoreAdjustments: Map[Long, Double] ): ThriftRankingParams = ThriftRankingParams( `type` = Some(ThriftScoringFunctionType.ModelBased), selectedModels = Some(Map(RealGraphScoringModel -> 1.0)), applyBoosts = false, authorSpecificScoreAdjustments = Some(authorSpecificScoreAdjustments) ) private def getRelevanceOptions( authorSpecificScoreAdjustments: Map[Long, Double], ): ThriftSearchRelevanceOptions = { ThriftSearchRelevanceOptions( maxConsecutiveSameUser = Some(MaxConsecutiveSameUser), rankingParams = Some(getModelBasedRankingParams(authorSpecificScoreAdjustments)), maxHitsToProcess = Some(MaxHitsToProcess), orderByRelevance = true ) } private def getThriftSearchQuery(query: EarlybirdModelBasedSearchQuery): ThriftSearchQuery = ThriftSearchQuery( serializedQuery = Some(f"(* [since_time ${query.oldestTweetTimestampInSec.getOrElse(0)}])"), fromUserIDFilter64 = Some(query.seedUserIds), numResults = query.maxNumTweets, maxHitsToProcess = MaxHitsToProcess, rankingMode = ThriftSearchRankingMode.Relevance, relevanceOptions = Some(getRelevanceOptions(query.frsUserToScoresForScoreAdjustment.getOrElse(Map.empty))), facetFieldNames = Some(FacetsToFetch), resultMetadataOptions = Some(MetadataOptions), searcherId = None ) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdRecencyBasedSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TweetWithAuthor import com.twitter.cr_mixer.similarity_engine.EarlybirdRecencyBasedSimilarityEngine.EarlybirdRecencyBasedSearchQuery import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.common.UserId import com.twitter.snowflake.id.SnowflakeId import com.twitter.storehaus.ReadableStore import com.twitter.util.Duration import com.twitter.util.Future import com.twitter.util.Time import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton case class EarlybirdRecencyBasedSimilarityEngine @Inject() ( @Named(ModuleNames.EarlybirdRecencyBasedWithoutRetweetsRepliesTweetsCache) earlybirdRecencyBasedWithoutRetweetsRepliesTweetsCacheStore: ReadableStore[ UserId, Seq[TweetId] ], @Named(ModuleNames.EarlybirdRecencyBasedWithRetweetsRepliesTweetsCache) earlybirdRecencyBasedWithRetweetsRepliesTweetsCacheStore: ReadableStore[ UserId, Seq[TweetId] ], timeoutConfig: TimeoutConfig, stats: StatsReceiver) extends ReadableStore[EarlybirdRecencyBasedSearchQuery, Seq[TweetWithAuthor]] { import EarlybirdRecencyBasedSimilarityEngine._ val statsReceiver: StatsReceiver = stats.scope(this.getClass.getSimpleName) override def get( query: EarlybirdRecencyBasedSearchQuery ): Future[Option[Seq[TweetWithAuthor]]] = { Future .collect { if (query.filterOutRetweetsAndReplies) { query.seedUserIds.map { seedUserId => StatsUtil.trackOptionItemsStats(statsReceiver.scope("WithoutRetweetsAndReplies")) { earlybirdRecencyBasedWithoutRetweetsRepliesTweetsCacheStore .get(seedUserId).map(_.map(_.map(tweetId => TweetWithAuthor(tweetId = tweetId, authorId = seedUserId)))) } } } else { query.seedUserIds.map { seedUserId => StatsUtil.trackOptionItemsStats(statsReceiver.scope("WithRetweetsAndReplies")) { earlybirdRecencyBasedWithRetweetsRepliesTweetsCacheStore .get(seedUserId) .map(_.map(_.map(tweetId => TweetWithAuthor(tweetId = tweetId, authorId = seedUserId)))) } } } } .map { tweetWithAuthorList => val earliestTweetId = SnowflakeId.firstIdFor(Time.now - query.maxTweetAge) tweetWithAuthorList .flatMap(_.getOrElse(Seq.empty)) .filter(tweetWithAuthor => tweetWithAuthor.tweetId >= earliestTweetId // tweet age filter && !query.excludedTweetIds .contains(tweetWithAuthor.tweetId)) // excluded tweet filter .sortBy(tweetWithAuthor => -SnowflakeId.unixTimeMillisFromId(tweetWithAuthor.tweetId)) // sort by recency .take(query.maxNumTweets) // take most recent N tweets } .map(result => Some(result)) } } object EarlybirdRecencyBasedSimilarityEngine { case class EarlybirdRecencyBasedSearchQuery( seedUserIds: Seq[UserId], maxNumTweets: Int, excludedTweetIds: Set[TweetId], maxTweetAge: Duration, filterOutRetweetsAndReplies: Boolean) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.TweetWithAuthor import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.storehaus.ReadableStore import com.twitter.util.Future class EarlybirdSimilarityEngine[ Query, EarlybirdSimilarityEngineStore <: ReadableStore[Query, Seq[TweetWithAuthor]] ]( implementingStore: EarlybirdSimilarityEngineStore, override val identifier: SimilarityEngineType, globalStats: StatsReceiver, engineConfig: SimilarityEngineConfig, ) extends SimilarityEngine[EngineQuery[Query], TweetWithAuthor] { private val scopedStats = globalStats.scope("similarityEngine", identifier.toString) def getScopedStats: StatsReceiver = scopedStats def getCandidates(query: EngineQuery[Query]): Future[Option[Seq[TweetWithAuthor]]] = { SimilarityEngine.getFromFn( implementingStore.get, query.storeQuery, engineConfig, query.params, scopedStats ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngineBase.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.TweetWithAuthor import com.twitter.finagle.stats.StatsReceiver import com.twitter.search.earlybird.thriftscala.EarlybirdRequest import com.twitter.search.earlybird.thriftscala.EarlybirdResponseCode import com.twitter.search.earlybird.thriftscala.EarlybirdService import com.twitter.simclusters_v2.common.UserId import com.twitter.storehaus.ReadableStore import com.twitter.util.Future /** * This trait is a base trait for Earlybird similarity engines. All Earlybird similarity * engines extend from it and override the construction method for EarlybirdRequest */ trait EarlybirdSimilarityEngineBase[EarlybirdSearchQuery] extends ReadableStore[EarlybirdSearchQuery, Seq[TweetWithAuthor]] { def earlybirdSearchClient: EarlybirdService.MethodPerEndpoint def statsReceiver: StatsReceiver def getEarlybirdRequest(query: EarlybirdSearchQuery): Option[EarlybirdRequest] override def get(query: EarlybirdSearchQuery): Future[Option[Seq[TweetWithAuthor]]] = { getEarlybirdRequest(query) .map { earlybirdRequest => earlybirdSearchClient .search(earlybirdRequest).map { response => response.responseCode match { case EarlybirdResponseCode.Success => val earlybirdSearchResult = response.searchResults .map( _.results .map(searchResult => TweetWithAuthor( searchResult.id, // fromUserId should be there since MetadataOptions.getFromUserId = true searchResult.metadata.map(_.fromUserId).getOrElse(0))).toSeq) statsReceiver.scope("result").stat("size").add(earlybirdSearchResult.size) earlybirdSearchResult case e => statsReceiver.scope("failures").counter(e.getClass.getSimpleName).incr() Some(Seq.empty) } } }.getOrElse(Future.None) } } object EarlybirdSimilarityEngineBase { trait EarlybirdSearchQuery { def seedUserIds: Seq[UserId] def maxNumTweets: Int } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngineRouter.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.EarlybirdSimilarityEngineType import com.twitter.cr_mixer.model.EarlybirdSimilarityEngineType_ModelBased import com.twitter.cr_mixer.model.EarlybirdSimilarityEngineType_RecencyBased import com.twitter.cr_mixer.model.EarlybirdSimilarityEngineType_TensorflowBased import com.twitter.cr_mixer.model.TweetWithAuthor import com.twitter.cr_mixer.param.EarlybirdFrsBasedCandidateGenerationParams import com.twitter.cr_mixer.param.EarlybirdFrsBasedCandidateGenerationParams.FrsBasedCandidateGenerationEarlybirdSimilarityEngineTypeParam import com.twitter.cr_mixer.param.FrsParams.FrsBasedCandidateGenerationMaxCandidatesNumParam import com.twitter.finagle.stats.StatsReceiver import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.common.UserId import com.twitter.snowflake.id.SnowflakeId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.util.Duration import com.twitter.util.Future import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton @Singleton case class EarlybirdSimilarityEngineRouter @Inject() ( earlybirdRecencyBasedSimilarityEngine: EarlybirdSimilarityEngine[ EarlybirdRecencyBasedSimilarityEngine.EarlybirdRecencyBasedSearchQuery, EarlybirdRecencyBasedSimilarityEngine ], earlybirdModelBasedSimilarityEngine: EarlybirdSimilarityEngine[ EarlybirdModelBasedSimilarityEngine.EarlybirdModelBasedSearchQuery, EarlybirdModelBasedSimilarityEngine ], earlybirdTensorflowBasedSimilarityEngine: EarlybirdSimilarityEngine[ EarlybirdTensorflowBasedSimilarityEngine.EarlybirdTensorflowBasedSearchQuery, EarlybirdTensorflowBasedSimilarityEngine ], timeoutConfig: TimeoutConfig, statsReceiver: StatsReceiver) extends ReadableStore[EarlybirdSimilarityEngineRouter.Query, Seq[TweetWithAuthor]] { import EarlybirdSimilarityEngineRouter._ override def get( k: EarlybirdSimilarityEngineRouter.Query ): Future[Option[Seq[TweetWithAuthor]]] = { k.rankingMode match { case EarlybirdSimilarityEngineType_RecencyBased => earlybirdRecencyBasedSimilarityEngine.getCandidates(recencyBasedQueryFromParams(k)) case EarlybirdSimilarityEngineType_ModelBased => earlybirdModelBasedSimilarityEngine.getCandidates(modelBasedQueryFromParams(k)) case EarlybirdSimilarityEngineType_TensorflowBased => earlybirdTensorflowBasedSimilarityEngine.getCandidates(tensorflowBasedQueryFromParams(k)) } } } object EarlybirdSimilarityEngineRouter { case class Query( searcherUserId: Option[UserId], seedUserIds: Seq[UserId], maxNumTweets: Int, excludedTweetIds: Set[TweetId], rankingMode: EarlybirdSimilarityEngineType, frsUserToScoresForScoreAdjustment: Option[Map[UserId, Double]], maxTweetAge: Duration, filterOutRetweetsAndReplies: Boolean, params: configapi.Params) def queryFromParams( searcherUserId: Option[UserId], seedUserIds: Seq[UserId], excludedTweetIds: Set[TweetId], frsUserToScoresForScoreAdjustment: Option[Map[UserId, Double]], params: configapi.Params ): Query = Query( searcherUserId, seedUserIds, maxNumTweets = params(FrsBasedCandidateGenerationMaxCandidatesNumParam), excludedTweetIds, rankingMode = params(FrsBasedCandidateGenerationEarlybirdSimilarityEngineTypeParam).rankingMode, frsUserToScoresForScoreAdjustment, maxTweetAge = params( EarlybirdFrsBasedCandidateGenerationParams.FrsBasedCandidateGenerationEarlybirdMaxTweetAge), filterOutRetweetsAndReplies = params( EarlybirdFrsBasedCandidateGenerationParams.FrsBasedCandidateGenerationEarlybirdFilterOutRetweetsAndReplies), params ) private def recencyBasedQueryFromParams( query: Query ): EngineQuery[EarlybirdRecencyBasedSimilarityEngine.EarlybirdRecencyBasedSearchQuery] = EngineQuery( EarlybirdRecencyBasedSimilarityEngine.EarlybirdRecencyBasedSearchQuery( seedUserIds = query.seedUserIds, maxNumTweets = query.maxNumTweets, excludedTweetIds = query.excludedTweetIds, maxTweetAge = query.maxTweetAge, filterOutRetweetsAndReplies = query.filterOutRetweetsAndReplies ), query.params ) private def tensorflowBasedQueryFromParams( query: Query, ): EngineQuery[EarlybirdTensorflowBasedSimilarityEngine.EarlybirdTensorflowBasedSearchQuery] = EngineQuery( EarlybirdTensorflowBasedSimilarityEngine.EarlybirdTensorflowBasedSearchQuery( searcherUserId = query.searcherUserId, seedUserIds = query.seedUserIds, maxNumTweets = query.maxNumTweets, // hard code the params below for now. Will move to FS after shipping the ddg beforeTweetIdExclusive = None, afterTweetIdExclusive = Some(SnowflakeId.firstIdFor((Time.now - query.maxTweetAge).inMilliseconds)), filterOutRetweetsAndReplies = query.filterOutRetweetsAndReplies, useTensorflowRanking = true, excludedTweetIds = query.excludedTweetIds, maxNumHitsPerShard = 1000 ), query.params ) private def modelBasedQueryFromParams( query: Query, ): EngineQuery[EarlybirdModelBasedSimilarityEngine.EarlybirdModelBasedSearchQuery] = EngineQuery( EarlybirdModelBasedSimilarityEngine.EarlybirdModelBasedSearchQuery( seedUserIds = query.seedUserIds, maxNumTweets = query.maxNumTweets, oldestTweetTimestampInSec = Some(query.maxTweetAge.ago.inSeconds), frsUserToScoresForScoreAdjustment = query.frsUserToScoresForScoreAdjustment ), query.params ) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdTensorflowBasedSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.finagle.stats.StatsReceiver import com.twitter.search.earlybird.thriftscala.EarlybirdRequest import com.twitter.search.earlybird.thriftscala.EarlybirdService import com.twitter.search.earlybird.thriftscala.ThriftSearchQuery import com.twitter.util.Time import com.twitter.search.common.query.thriftjava.thriftscala.CollectorParams import com.twitter.search.common.ranking.thriftscala.ThriftRankingParams import com.twitter.search.common.ranking.thriftscala.ThriftScoringFunctionType import com.twitter.search.earlybird.thriftscala.ThriftSearchRelevanceOptions import javax.inject.Inject import javax.inject.Singleton import EarlybirdSimilarityEngineBase._ import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.similarity_engine.EarlybirdTensorflowBasedSimilarityEngine.EarlybirdTensorflowBasedSearchQuery import com.twitter.cr_mixer.util.EarlybirdSearchUtil.EarlybirdClientId import com.twitter.cr_mixer.util.EarlybirdSearchUtil.FacetsToFetch import com.twitter.cr_mixer.util.EarlybirdSearchUtil.GetCollectorTerminationParams import com.twitter.cr_mixer.util.EarlybirdSearchUtil.GetEarlybirdQuery import com.twitter.cr_mixer.util.EarlybirdSearchUtil.MetadataOptions import com.twitter.cr_mixer.util.EarlybirdSearchUtil.GetNamedDisjunctions import com.twitter.search.earlybird.thriftscala.ThriftSearchRankingMode import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.common.UserId import com.twitter.util.Duration @Singleton case class EarlybirdTensorflowBasedSimilarityEngine @Inject() ( earlybirdSearchClient: EarlybirdService.MethodPerEndpoint, timeoutConfig: TimeoutConfig, stats: StatsReceiver) extends EarlybirdSimilarityEngineBase[EarlybirdTensorflowBasedSearchQuery] { import EarlybirdTensorflowBasedSimilarityEngine._ override val statsReceiver: StatsReceiver = stats.scope(this.getClass.getSimpleName) override def getEarlybirdRequest( query: EarlybirdTensorflowBasedSearchQuery ): Option[EarlybirdRequest] = { if (query.seedUserIds.nonEmpty) Some( EarlybirdRequest( searchQuery = getThriftSearchQuery(query, timeoutConfig.earlybirdServerTimeout), clientHost = None, clientRequestID = None, clientId = Some(EarlybirdClientId), clientRequestTimeMs = Some(Time.now.inMilliseconds), cachingParams = None, timeoutMs = timeoutConfig.earlybirdServerTimeout.inMilliseconds.intValue(), facetRequest = None, termStatisticsRequest = None, debugMode = 0, debugOptions = None, searchSegmentId = None, returnStatusType = None, successfulResponseThreshold = None, querySource = None, getOlderResults = Some(false), followedUserIds = Some(query.seedUserIds), adjustedProtectedRequestParams = None, adjustedFullArchiveRequestParams = None, getProtectedTweetsOnly = Some(false), retokenizeSerializedQuery = None, skipVeryRecentTweets = true, experimentClusterToUse = None )) else None } } object EarlybirdTensorflowBasedSimilarityEngine { case class EarlybirdTensorflowBasedSearchQuery( searcherUserId: Option[UserId], seedUserIds: Seq[UserId], maxNumTweets: Int, beforeTweetIdExclusive: Option[TweetId], afterTweetIdExclusive: Option[TweetId], filterOutRetweetsAndReplies: Boolean, useTensorflowRanking: Boolean, excludedTweetIds: Set[TweetId], maxNumHitsPerShard: Int) extends EarlybirdSearchQuery private def getThriftSearchQuery( query: EarlybirdTensorflowBasedSearchQuery, processingTimeout: Duration ): ThriftSearchQuery = ThriftSearchQuery( serializedQuery = GetEarlybirdQuery( query.beforeTweetIdExclusive, query.afterTweetIdExclusive, query.excludedTweetIds, query.filterOutRetweetsAndReplies).map(_.serialize), fromUserIDFilter64 = Some(query.seedUserIds), numResults = query.maxNumTweets, // Whether to collect conversation IDs. Remove it for now. // collectConversationId = Gate.True(), // true for Home rankingMode = ThriftSearchRankingMode.Relevance, relevanceOptions = Some(getRelevanceOptions), collectorParams = Some( CollectorParams( // numResultsToReturn defines how many results each EB shard will return to search root numResultsToReturn = 1000, // terminationParams.maxHitsToProcess is used for early terminating per shard results fetching. terminationParams = GetCollectorTerminationParams(query.maxNumHitsPerShard, processingTimeout) )), facetFieldNames = Some(FacetsToFetch), resultMetadataOptions = Some(MetadataOptions), searcherId = query.searcherUserId, searchStatusIds = None, namedDisjunctionMap = GetNamedDisjunctions(query.excludedTweetIds) ) // The specific values of recap relevance/reranking options correspond to // experiment: enable_recap_reranking_2988,timeline_internal_disable_recap_filter // bucket : enable_rerank,disable_filter private def getRelevanceOptions: ThriftSearchRelevanceOptions = { ThriftSearchRelevanceOptions( proximityScoring = true, maxConsecutiveSameUser = Some(2), rankingParams = Some(getTensorflowBasedRankingParams), maxHitsToProcess = Some(500), maxUserBlendCount = Some(3), proximityPhraseWeight = 9.0, returnAllResults = Some(true) ) } private def getTensorflowBasedRankingParams: ThriftRankingParams = { ThriftRankingParams( `type` = Some(ThriftScoringFunctionType.TensorflowBased), selectedTensorflowModel = Some("timelines_rectweet_replica"), minScore = -1.0e100, applyBoosts = false, authorSpecificScoreAdjustments = None ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/FilterUtil.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.snowflake.id.SnowflakeId import com.twitter.util.Duration import com.twitter.util.Time object FilterUtil { /** Returns a list of tweets that are generated less than `maxTweetAgeHours` hours ago */ def tweetAgeFilter( candidates: Seq[TweetWithScore], maxTweetAgeHours: Duration ): Seq[TweetWithScore] = { // Tweet IDs are approximately chronological (see http://go/snowflake), // so we are building the earliest tweet id once // The per-candidate logic here then be candidate.tweetId > earliestPermittedTweetId, which is far cheaper. // See @cyao's phab on CrMixer generic age filter for reference https://phabricator.twitter.biz/D903188 val earliestTweetId = SnowflakeId.firstIdFor(Time.now - maxTweetAgeHours) candidates.filter { candidate => candidate.tweetId >= earliestTweetId } } /** Returns a list of tweet sources that are generated less than `maxTweetAgeHours` hours ago */ def tweetSourceAgeFilter( candidates: Seq[SourceInfo], maxTweetSignalAgeHoursParam: Duration ): Seq[SourceInfo] = { // Tweet IDs are approximately chronological (see http://go/snowflake), // so we are building the earliest tweet id once // This filter applies to source signals. Some candidate source calls can be avoided if source signals // can be filtered. val earliestTweetId = SnowflakeId.firstIdFor(Time.now - maxTweetSignalAgeHoursParam) candidates.filter { candidate => candidate.internalId match { case InternalId.TweetId(tweetId) => tweetId >= earliestTweetId case _ => false } } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/HnswANNSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.ann.common.thriftscala.AnnQueryService import com.twitter.ann.common.thriftscala.Distance import com.twitter.ann.common.thriftscala.NearestNeighborQuery import com.twitter.ann.hnsw.HnswCommon import com.twitter.ann.hnsw.HnswParams import com.twitter.bijection.Injection import com.twitter.cortex.ml.embeddings.common.TweetKind import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.MemCacheConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.mediaservices.commons.codec.ArrayByteBufferCodec import com.twitter.ml.api.thriftscala.{Embedding => ThriftEmbedding} import com.twitter.ml.featurestore.lib import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi.Params import com.twitter.util.Future case class HnswANNEngineQuery( modelId: String, sourceId: InternalId, params: Params, ) { val cacheKey: String = s"${modelId}_${sourceId.toString}" } /** * This Engine looks for tweets whose similarity is close to a Source Dense Embedding. * Only support Long based embedding lookup. UserId or TweetId. * * It provides HNSW specific implementations * * @param memCacheConfigOpt If specified, it will wrap the underlying store with a MemCache layer * You should only enable this for cacheable queries, e.x. TweetIds. * consumer based UserIds are generally not possible to cache. */ class HnswANNSimilarityEngine( embeddingStoreLookUpMap: Map[String, ReadableStore[InternalId, ThriftEmbedding]], annServiceLookUpMap: Map[String, AnnQueryService.MethodPerEndpoint], globalStats: StatsReceiver, override val identifier: SimilarityEngineType, engineConfig: SimilarityEngineConfig, memCacheConfigOpt: Option[MemCacheConfig[HnswANNEngineQuery]] = None) extends SimilarityEngine[HnswANNEngineQuery, TweetWithScore] { private val MaxNumResults: Int = 200 private val ef: Int = 800 private val TweetIdByteInjection: Injection[lib.TweetId, Array[Byte]] = TweetKind.byteInjection private val scopedStats = globalStats.scope("similarityEngine", identifier.toString) def getScopedStats: StatsReceiver = scopedStats private def fetchEmbedding( query: HnswANNEngineQuery, ): Future[Option[ThriftEmbedding]] = { val embeddingStore = embeddingStoreLookUpMap.getOrElse( query.modelId, throw new IllegalArgumentException( s"${this.getClass.getSimpleName} ${identifier.toString}: " + s"ModelId ${query.modelId} does not exist for embeddingStore" ) ) embeddingStore.get(query.sourceId) } private def fetchCandidates( query: HnswANNEngineQuery, embedding: ThriftEmbedding ): Future[Seq[TweetWithScore]] = { val annService = annServiceLookUpMap.getOrElse( query.modelId, throw new IllegalArgumentException( s"${this.getClass.getSimpleName} ${identifier.toString}: " + s"ModelId ${query.modelId} does not exist for annStore" ) ) val hnswParams = HnswCommon.RuntimeParamsInjection.apply(HnswParams(ef)) val annQuery = NearestNeighborQuery(embedding, withDistance = true, hnswParams, MaxNumResults) annService .query(annQuery) .map( _.nearestNeighbors .map { nearestNeighbor => val candidateId = TweetIdByteInjection .invert(ArrayByteBufferCodec.decode(nearestNeighbor.id)) .toOption .map(_.tweetId) (candidateId, nearestNeighbor.distance) }.collect { case (Some(candidateId), Some(distance)) => TweetWithScore(candidateId, toScore(distance)) }) } // Convert Distance to a score such that higher scores mean more similar. def toScore(distance: Distance): Double = { distance match { case Distance.EditDistance(editDistance) => // (-Infinite, 0.0] 0.0 - editDistance.distance case Distance.L2Distance(l2Distance) => // (-Infinite, 0.0] 0.0 - l2Distance.distance case Distance.CosineDistance(cosineDistance) => // [0.0 - 1.0] 1.0 - cosineDistance.distance case Distance.InnerProductDistance(innerProductDistance) => // (-Infinite, Infinite) 1.0 - innerProductDistance.distance case Distance.UnknownUnionField(_) => throw new IllegalStateException( s"${this.getClass.getSimpleName} does not recognize $distance.toString" ) } } private[similarity_engine] def getEmbeddingAndCandidates( query: HnswANNEngineQuery ): Future[Option[Seq[TweetWithScore]]] = { val fetchEmbeddingStat = scopedStats.scope(query.modelId).scope("fetchEmbedding") val fetchCandidatesStat = scopedStats.scope(query.modelId).scope("fetchCandidates") for { embeddingOpt <- StatsUtil.trackOptionStats(fetchEmbeddingStat) { fetchEmbedding(query) } candidates <- StatsUtil.trackItemsStats(fetchCandidatesStat) { embeddingOpt match { case Some(embedding) => fetchCandidates(query, embedding) case None => Future.Nil } } } yield { Some(candidates) } } // Add memcache wrapper, if specified private val store = { val uncachedStore = ReadableStore.fromFnFuture(getEmbeddingAndCandidates) memCacheConfigOpt match { case Some(config) => SimilarityEngine.addMemCache( underlyingStore = uncachedStore, memCacheConfig = config, statsReceiver = scopedStats ) case _ => uncachedStore } } def toSimilarityEngineInfo( query: HnswANNEngineQuery, score: Double ): SimilarityEngineInfo = { SimilarityEngineInfo( similarityEngineType = this.identifier, modelId = Some(query.modelId), score = Some(score)) } override def getCandidates( engineQuery: HnswANNEngineQuery ): Future[Option[Seq[TweetWithScore]]] = { val versionedStats = globalStats.scope(engineQuery.modelId) SimilarityEngine.getFromFn( store.get, engineQuery, engineConfig, engineQuery.params, versionedStats ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/LookupSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.MemCacheConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi.Params import com.twitter.util.Future case class LookupEngineQuery[Query]( storeQuery: Query, // the actual Query type of the underlying store lookupKey: String, params: Params, ) /** * This Engine provides a map interface for looking up different model implementations. * It provides modelId level monitoring for free. * * Example use cases include OfflineSimClusters lookup * * * @param versionedStoreMap A mapping from a modelId to a corresponding implementation * @param memCacheConfigOpt If specified, it will wrap the underlying store with a MemCache layer * You should only enable this for cacheable queries, e.x. TweetIds. * consumer based UserIds are generally not possible to cache. */ class LookupSimilarityEngine[Query, Candidate <: Serializable]( versionedStoreMap: Map[String, ReadableStore[Query, Seq[Candidate]]], // key = modelId override val identifier: SimilarityEngineType, globalStats: StatsReceiver, engineConfig: SimilarityEngineConfig, memCacheConfigOpt: Option[MemCacheConfig[Query]] = None) extends SimilarityEngine[LookupEngineQuery[Query], Candidate] { private val scopedStats = globalStats.scope("similarityEngine", identifier.toString) private val underlyingLookupMap = { memCacheConfigOpt match { case Some(config) => versionedStoreMap.map { case (modelId, store) => ( modelId, SimilarityEngine.addMemCache( underlyingStore = store, memCacheConfig = config, keyPrefix = Some(modelId), statsReceiver = scopedStats ) ) } case _ => versionedStoreMap } } override def getCandidates( engineQuery: LookupEngineQuery[Query] ): Future[Option[Seq[Candidate]]] = { val versionedStore = underlyingLookupMap .getOrElse( engineQuery.lookupKey, throw new IllegalArgumentException( s"${this.getClass.getSimpleName} ${identifier.toString}: ModelId ${engineQuery.lookupKey} does not exist" ) ) SimilarityEngine.getFromFn( fn = versionedStore.get, storeQuery = engineQuery.storeQuery, engineConfig = engineConfig, params = engineQuery.params, scopedStats = scopedStats.scope(engineQuery.lookupKey) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ModelBasedANNStore.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.ann.common.thriftscala.AnnQueryService import com.twitter.ann.common.thriftscala.Distance import com.twitter.ann.common.thriftscala.NearestNeighborQuery import com.twitter.ann.common.thriftscala.NearestNeighborResult import com.twitter.ann.hnsw.HnswCommon import com.twitter.ann.hnsw.HnswParams import com.twitter.bijection.Injection import com.twitter.conversions.DurationOps._ import com.twitter.cortex.ml.embeddings.common.TweetKind import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.mediaservices.commons.codec.ArrayByteBufferCodec import com.twitter.ml.api.thriftscala.{Embedding => ThriftEmbedding} import com.twitter.ml.featurestore.lib import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.util.Duration import com.twitter.util.Future import javax.inject.Singleton /** * This store looks for tweets whose similarity is close to a Source Dense Embedding. * Only support Long based embedding lookup. UserId or TweetId */ @Singleton class ModelBasedANNStore( embeddingStoreLookUpMap: Map[String, ReadableStore[InternalId, ThriftEmbedding]], annServiceLookUpMap: Map[String, AnnQueryService.MethodPerEndpoint], globalStats: StatsReceiver) extends ReadableStore[ ModelBasedANNStore.Query, Seq[TweetWithScore] ] { import ModelBasedANNStore._ private val stats = globalStats.scope(this.getClass.getSimpleName) private val fetchEmbeddingStat = stats.scope("fetchEmbedding") private val fetchCandidatesStat = stats.scope("fetchCandidates") override def get(query: Query): Future[Option[Seq[TweetWithScore]]] = { for { maybeEmbedding <- StatsUtil.trackOptionStats(fetchEmbeddingStat.scope(query.modelId)) { fetchEmbedding(query) } maybeCandidates <- StatsUtil.trackOptionStats(fetchCandidatesStat.scope(query.modelId)) { maybeEmbedding match { case Some(embedding) => fetchCandidates(query, embedding) case None => Future.None } } } yield { maybeCandidates.map( _.nearestNeighbors .map { nearestNeighbor => val candidateId = TweetIdByteInjection .invert(ArrayByteBufferCodec.decode(nearestNeighbor.id)) .toOption .map(_.tweetId) (candidateId, nearestNeighbor.distance) }.collect { case (Some(candidateId), Some(distance)) => TweetWithScore(candidateId, toScore(distance)) }) } } private def fetchEmbedding(query: Query): Future[Option[ThriftEmbedding]] = { embeddingStoreLookUpMap.get(query.modelId) match { case Some(embeddingStore) => embeddingStore.get(query.sourceId) case _ => Future.None } } private def fetchCandidates( query: Query, embedding: ThriftEmbedding ): Future[Option[NearestNeighborResult]] = { val hnswParams = HnswCommon.RuntimeParamsInjection.apply(HnswParams(query.ef)) annServiceLookUpMap.get(query.modelId) match { case Some(annService) => val annQuery = NearestNeighborQuery(embedding, withDistance = true, hnswParams, MaxNumResults) annService.query(annQuery).map(v => Some(v)) case _ => Future.None } } } object ModelBasedANNStore { val MaxNumResults: Int = 200 val MaxTweetCandidateAge: Duration = 1.day val TweetIdByteInjection: Injection[lib.TweetId, Array[Byte]] = TweetKind.byteInjection // For more information about HNSW algorithm: https://docbird.twitter.biz/ann/hnsw.html case class Query( sourceId: InternalId, modelId: String, similarityEngineType: SimilarityEngineType, ef: Int = 800) def toScore(distance: Distance): Double = { distance match { case Distance.L2Distance(l2Distance) => // (-Infinite, 0.0] 0.0 - l2Distance.distance case Distance.CosineDistance(cosineDistance) => // [0.0 - 1.0] 1.0 - cosineDistance.distance case Distance.InnerProductDistance(innerProductDistance) => // (-Infinite, Infinite) 1.0 - innerProductDistance.distance case _ => 0.0 } } def toSimilarityEngineInfo(query: Query, score: Double): SimilarityEngineInfo = { SimilarityEngineInfo( similarityEngineType = query.similarityEngineType, modelId = Some(query.modelId), score = Some(score)) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUnifiedSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.CandidateGenerationInfo import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.model.TweetWithCandidateGenerationInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.param.ProducerBasedCandidateGenerationParams import com.twitter.cr_mixer.param.UnifiedSETweetCombinationMethod import com.twitter.cr_mixer.param.RelatedTweetProducerBasedParams import com.twitter.cr_mixer.param.SimClustersANNParams import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.cr_mixer.util.InterleaveUtil import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.simclusters_v2.common.ModelVersions import com.twitter.simclusters_v2.thriftscala.EmbeddingType import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.util.Duration import com.twitter.util.Future import javax.inject.Named import javax.inject.Singleton import scala.collection.mutable.ArrayBuffer /** * This store looks for similar tweets from UserTweetGraph for a Source ProducerId * For a query producerId,User Tweet Graph (UTG), * lets us find out which tweets the query producer's followers co-engaged */ @Singleton case class ProducerBasedUnifiedSimilarityEngine( @Named(ModuleNames.ProducerBasedUserTweetGraphSimilarityEngine) producerBasedUserTweetGraphSimilarityEngine: StandardSimilarityEngine[ ProducerBasedUserTweetGraphSimilarityEngine.Query, TweetWithScore ], simClustersANNSimilarityEngine: StandardSimilarityEngine[ SimClustersANNSimilarityEngine.Query, TweetWithScore ], statsReceiver: StatsReceiver) extends ReadableStore[ProducerBasedUnifiedSimilarityEngine.Query, Seq[ TweetWithCandidateGenerationInfo ]] { import ProducerBasedUnifiedSimilarityEngine._ private val stats = statsReceiver.scope(this.getClass.getSimpleName) private val fetchCandidatesStat = stats.scope("fetchCandidates") override def get( query: Query ): Future[Option[Seq[TweetWithCandidateGenerationInfo]]] = { query.sourceInfo.internalId match { case _: InternalId.UserId => StatsUtil.trackOptionItemsStats(fetchCandidatesStat) { val sannCandidatesFut = if (query.enableSimClustersANN) { simClustersANNSimilarityEngine.getCandidates(query.simClustersANNQuery) } else Future.None val sann1CandidatesFut = if (query.enableSimClustersANN1) { simClustersANNSimilarityEngine.getCandidates(query.simClustersANN1Query) } else Future.None val sann2CandidatesFut = if (query.enableSimClustersANN2) { simClustersANNSimilarityEngine.getCandidates(query.simClustersANN2Query) } else Future.None val sann3CandidatesFut = if (query.enableSimClustersANN3) { simClustersANNSimilarityEngine.getCandidates(query.simClustersANN3Query) } else Future.None val sann4CandidatesFut = if (query.enableSimClustersANN4) { simClustersANNSimilarityEngine.getCandidates(query.simClustersANN4Query) } else Future.None val sann5CandidatesFut = if (query.enableSimClustersANN5) { simClustersANNSimilarityEngine.getCandidates(query.simClustersANN5Query) } else Future.None val experimentalSANNCandidatesFut = if (query.enableExperimentalSimClustersANN) { simClustersANNSimilarityEngine.getCandidates(query.experimentalSimClustersANNQuery) } else Future.None val utgCandidatesFut = if (query.enableUtg) { producerBasedUserTweetGraphSimilarityEngine.getCandidates(query.utgQuery) } else Future.None Future .join( sannCandidatesFut, sann1CandidatesFut, sann2CandidatesFut, sann3CandidatesFut, sann4CandidatesFut, sann5CandidatesFut, experimentalSANNCandidatesFut, utgCandidatesFut ).map { case ( simClustersAnnCandidates, simClustersAnn1Candidates, simClustersAnn2Candidates, simClustersAnn3Candidates, simClustersAnn4Candidates, simClustersAnn5Candidates, experimentalSANNCandidates, userTweetGraphCandidates) => val filteredSANNTweets = simClustersCandidateMinScoreFilter( simClustersAnnCandidates.toSeq.flatten, query.simClustersMinScore, query.simClustersANNQuery.storeQuery.simClustersANNConfigId) val filteredExperimentalSANNTweets = simClustersCandidateMinScoreFilter( experimentalSANNCandidates.toSeq.flatten, query.simClustersMinScore, query.experimentalSimClustersANNQuery.storeQuery.simClustersANNConfigId) val filteredSANN1Tweets = simClustersCandidateMinScoreFilter( simClustersAnn1Candidates.toSeq.flatten, query.simClustersMinScore, query.simClustersANN1Query.storeQuery.simClustersANNConfigId) val filteredSANN2Tweets = simClustersCandidateMinScoreFilter( simClustersAnn2Candidates.toSeq.flatten, query.simClustersMinScore, query.simClustersANN2Query.storeQuery.simClustersANNConfigId) val filteredSANN3Tweets = simClustersCandidateMinScoreFilter( simClustersAnn3Candidates.toSeq.flatten, query.simClustersMinScore, query.simClustersANN3Query.storeQuery.simClustersANNConfigId) val filteredSANN4Tweets = simClustersCandidateMinScoreFilter( simClustersAnn4Candidates.toSeq.flatten, query.simClustersMinScore, query.simClustersANN4Query.storeQuery.simClustersANNConfigId) val filteredSANN5Tweets = simClustersCandidateMinScoreFilter( simClustersAnn5Candidates.toSeq.flatten, query.simClustersMinScore, query.simClustersANN5Query.storeQuery.simClustersANNConfigId) val filteredUTGTweets = userTweetGraphFilter(userTweetGraphCandidates.toSeq.flatten) val sannTweetsWithCGInfo = filteredSANNTweets.map { tweetWithScore => val similarityEngineInfo = SimClustersANNSimilarityEngine .toSimilarityEngineInfo(query.simClustersANNQuery, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val sann1TweetsWithCGInfo = filteredSANN1Tweets.map { tweetWithScore => val similarityEngineInfo = SimClustersANNSimilarityEngine .toSimilarityEngineInfo(query.simClustersANN1Query, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val sann2TweetsWithCGInfo = filteredSANN2Tweets.map { tweetWithScore => val similarityEngineInfo = SimClustersANNSimilarityEngine .toSimilarityEngineInfo(query.simClustersANN2Query, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val sann3TweetsWithCGInfo = filteredSANN3Tweets.map { tweetWithScore => val similarityEngineInfo = SimClustersANNSimilarityEngine .toSimilarityEngineInfo(query.simClustersANN3Query, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val sann4TweetsWithCGInfo = filteredSANN4Tweets.map { tweetWithScore => val similarityEngineInfo = SimClustersANNSimilarityEngine .toSimilarityEngineInfo(query.simClustersANN4Query, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val sann5TweetsWithCGInfo = filteredSANN5Tweets.map { tweetWithScore => val similarityEngineInfo = SimClustersANNSimilarityEngine .toSimilarityEngineInfo(query.simClustersANN5Query, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val experimentalSANNTweetsWithCGInfo = filteredExperimentalSANNTweets.map { tweetWithScore => val similarityEngineInfo = SimClustersANNSimilarityEngine .toSimilarityEngineInfo( query.experimentalSimClustersANNQuery, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val utgTweetsWithCGInfo = filteredUTGTweets.map { tweetWithScore => val similarityEngineInfo = ProducerBasedUserTweetGraphSimilarityEngine .toSimilarityEngineInfo(tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val candidateSourcesToBeInterleaved = ArrayBuffer[Seq[TweetWithCandidateGenerationInfo]]( sannTweetsWithCGInfo, sann1TweetsWithCGInfo, sann2TweetsWithCGInfo, sann3TweetsWithCGInfo, sann4TweetsWithCGInfo, sann5TweetsWithCGInfo, experimentalSANNTweetsWithCGInfo, ) if (query.utgCombinationMethod == UnifiedSETweetCombinationMethod.Interleave) { candidateSourcesToBeInterleaved += utgTweetsWithCGInfo } val interleavedCandidates = InterleaveUtil.interleave(candidateSourcesToBeInterleaved) val candidateSourcesToBeOrdered = ArrayBuffer[Seq[TweetWithCandidateGenerationInfo]](interleavedCandidates) if (query.utgCombinationMethod == UnifiedSETweetCombinationMethod.Frontload) candidateSourcesToBeOrdered.prepend(utgTweetsWithCGInfo) val candidatesFromGivenOrderCombination = SimilaritySourceOrderingUtil.keepGivenOrder(candidateSourcesToBeOrdered) val unifiedCandidatesWithUnifiedCGInfo = candidatesFromGivenOrderCombination.map { candidate => /*** * when a candidate was made by interleave/keepGivenOrder, * then we apply getProducerBasedUnifiedCGInfo() to override with the unified CGInfo * * in contributingSE list for interleave. We only have the chosen SE available. * This is hard to add for interleave, and we plan to add it later after abstraction improvement. */ TweetWithCandidateGenerationInfo( tweetId = candidate.tweetId, candidateGenerationInfo = getProducerBasedUnifiedCGInfo( candidate.candidateGenerationInfo.sourceInfoOpt, candidate.getSimilarityScore, candidate.candidateGenerationInfo.contributingSimilarityEngines ) // getSimilarityScore comes from either unifiedScore or single score ) } stats.stat("unified_candidate_size").add(unifiedCandidatesWithUnifiedCGInfo.size) val truncatedCandidates = unifiedCandidatesWithUnifiedCGInfo.take(query.maxCandidateNumPerSourceKey) stats.stat("truncatedCandidates_size").add(truncatedCandidates.size) Some(truncatedCandidates) } } case _ => stats.counter("sourceId_is_not_userId_cnt").incr() Future.None } } private def simClustersCandidateMinScoreFilter( simClustersAnnCandidates: Seq[TweetWithScore], simClustersMinScore: Double, simClustersANNConfigId: String ): Seq[TweetWithScore] = { val filteredCandidates = simClustersAnnCandidates .filter { candidate => candidate.score > simClustersMinScore } stats.stat(simClustersANNConfigId, "simClustersAnnCandidates_size").add(filteredCandidates.size) stats.counter(simClustersANNConfigId, "simClustersAnnRequests").incr() if (filteredCandidates.isEmpty) stats.counter(simClustersANNConfigId, "emptyFilteredSimClustersAnnCandidates").incr() filteredCandidates.map { candidate => TweetWithScore(candidate.tweetId, candidate.score) } } /** A no-op filter as UTG filter already happened at UTG service side */ private def userTweetGraphFilter( userTweetGraphCandidates: Seq[TweetWithScore] ): Seq[TweetWithScore] = { val filteredCandidates = userTweetGraphCandidates stats.stat("userTweetGraphCandidates_size").add(userTweetGraphCandidates.size) if (filteredCandidates.isEmpty) stats.counter("emptyFilteredUserTweetGraphCandidates").incr() filteredCandidates.map { candidate => TweetWithScore(candidate.tweetId, candidate.score) } } } object ProducerBasedUnifiedSimilarityEngine { /*** * Every candidate will have the CG Info with ProducerBasedUnifiedSimilarityEngine * as they are generated by a composite of Similarity Engines. * Additionally, we store the contributing SEs (eg., SANN, UTG). */ private def getProducerBasedUnifiedCGInfo( sourceInfoOpt: Option[SourceInfo], unifiedScore: Double, contributingSimilarityEngines: Seq[SimilarityEngineInfo] ): CandidateGenerationInfo = { CandidateGenerationInfo( sourceInfoOpt, SimilarityEngineInfo( similarityEngineType = SimilarityEngineType.ProducerBasedUnifiedSimilarityEngine, modelId = None, // We do not assign modelId for a unified similarity engine score = Some(unifiedScore) ), contributingSimilarityEngines ) } case class Query( sourceInfo: SourceInfo, maxCandidateNumPerSourceKey: Int, maxTweetAgeHours: Duration, // SimClusters enableSimClustersANN: Boolean, simClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], enableExperimentalSimClustersANN: Boolean, experimentalSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], enableSimClustersANN1: Boolean, simClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query], enableSimClustersANN2: Boolean, simClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query], enableSimClustersANN4: Boolean, simClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query], enableSimClustersANN3: Boolean, simClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query], enableSimClustersANN5: Boolean, simClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query], simClustersMinScore: Double, // UTG enableUtg: Boolean, utgCombinationMethod: UnifiedSETweetCombinationMethod.Value, utgQuery: EngineQuery[ProducerBasedUserTweetGraphSimilarityEngine.Query]) def fromParams( sourceInfo: SourceInfo, params: configapi.Params, ): EngineQuery[Query] = { val maxCandidateNumPerSourceKey = params(GlobalParams.MaxCandidateNumPerSourceKeyParam) val maxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam) // SimClusters val enableSimClustersANN = params( ProducerBasedCandidateGenerationParams.EnableSimClustersANNParam) val simClustersModelVersion = ModelVersions.Enum.enumToSimClustersModelVersionMap(params(GlobalParams.ModelVersionParam)) val simClustersANNConfigId = params(SimClustersANNParams.SimClustersANNConfigId) // SimClusters - Experimental SANN Similarity Engine val enableExperimentalSimClustersANN = params( ProducerBasedCandidateGenerationParams.EnableExperimentalSimClustersANNParam) val experimentalSimClustersANNConfigId = params( SimClustersANNParams.ExperimentalSimClustersANNConfigId) // SimClusters - SANN cluster 1 Similarity Engine val enableSimClustersANN1 = params( ProducerBasedCandidateGenerationParams.EnableSimClustersANN1Param) val simClustersANN1ConfigId = params(SimClustersANNParams.SimClustersANN1ConfigId) // SimClusters - SANN cluster 2 Similarity Engine val enableSimClustersANN2 = params( ProducerBasedCandidateGenerationParams.EnableSimClustersANN2Param) val simClustersANN2ConfigId = params(SimClustersANNParams.SimClustersANN2ConfigId) // SimClusters - SANN cluster 3 Similarity Engine val enableSimClustersANN3 = params( ProducerBasedCandidateGenerationParams.EnableSimClustersANN3Param) val simClustersANN3ConfigId = params(SimClustersANNParams.SimClustersANN3ConfigId) // SimClusters - SANN cluster 5 Similarity Engine val enableSimClustersANN5 = params( ProducerBasedCandidateGenerationParams.EnableSimClustersANN5Param) val simClustersANN5ConfigId = params(SimClustersANNParams.SimClustersANN5ConfigId) val enableSimClustersANN4 = params( ProducerBasedCandidateGenerationParams.EnableSimClustersANN4Param) val simClustersANN4ConfigId = params(SimClustersANNParams.SimClustersANN4ConfigId) val simClustersMinScore = params( ProducerBasedCandidateGenerationParams.SimClustersMinScoreParam) // SimClusters ANN Query val simClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( sourceInfo.internalId, EmbeddingType.FavBasedProducer, simClustersModelVersion, simClustersANNConfigId, params ) val experimentalSimClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( sourceInfo.internalId, EmbeddingType.FavBasedProducer, simClustersModelVersion, experimentalSimClustersANNConfigId, params ) val simClustersANN1Query = SimClustersANNSimilarityEngine.fromParams( sourceInfo.internalId, EmbeddingType.FavBasedProducer, simClustersModelVersion, simClustersANN1ConfigId, params ) val simClustersANN2Query = SimClustersANNSimilarityEngine.fromParams( sourceInfo.internalId, EmbeddingType.FavBasedProducer, simClustersModelVersion, simClustersANN2ConfigId, params ) val simClustersANN3Query = SimClustersANNSimilarityEngine.fromParams( sourceInfo.internalId, EmbeddingType.FavBasedProducer, simClustersModelVersion, simClustersANN3ConfigId, params ) val simClustersANN5Query = SimClustersANNSimilarityEngine.fromParams( sourceInfo.internalId, EmbeddingType.FavBasedProducer, simClustersModelVersion, simClustersANN5ConfigId, params ) val simClustersANN4Query = SimClustersANNSimilarityEngine.fromParams( sourceInfo.internalId, EmbeddingType.FavBasedProducer, simClustersModelVersion, simClustersANN4ConfigId, params ) // UTG val enableUtg = params(ProducerBasedCandidateGenerationParams.EnableUTGParam) val utgCombinationMethod = params( ProducerBasedCandidateGenerationParams.UtgCombinationMethodParam) EngineQuery( Query( sourceInfo = sourceInfo, maxCandidateNumPerSourceKey = maxCandidateNumPerSourceKey, maxTweetAgeHours = maxTweetAgeHours, enableSimClustersANN = enableSimClustersANN, simClustersANNQuery = simClustersANNQuery, enableExperimentalSimClustersANN = enableExperimentalSimClustersANN, experimentalSimClustersANNQuery = experimentalSimClustersANNQuery, enableSimClustersANN1 = enableSimClustersANN1, simClustersANN1Query = simClustersANN1Query, enableSimClustersANN2 = enableSimClustersANN2, simClustersANN2Query = simClustersANN2Query, enableSimClustersANN3 = enableSimClustersANN3, simClustersANN3Query = simClustersANN3Query, enableSimClustersANN5 = enableSimClustersANN5, simClustersANN5Query = simClustersANN5Query, enableSimClustersANN4 = enableSimClustersANN4, simClustersANN4Query = simClustersANN4Query, simClustersMinScore = simClustersMinScore, enableUtg = enableUtg, utgCombinationMethod = utgCombinationMethod, utgQuery = ProducerBasedUserTweetGraphSimilarityEngine .fromParams(sourceInfo.internalId, params) ), params ) } def fromParamsForRelatedTweet( internalId: InternalId, params: configapi.Params ): EngineQuery[Query] = { val maxCandidateNumPerSourceKey = params(GlobalParams.MaxCandidateNumPerSourceKeyParam) val maxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam) // SimClusters val enableSimClustersANN = params(RelatedTweetProducerBasedParams.EnableSimClustersANNParam) val simClustersModelVersion = ModelVersions.Enum.enumToSimClustersModelVersionMap(params(GlobalParams.ModelVersionParam)) val simClustersANNConfigId = params(SimClustersANNParams.SimClustersANNConfigId) val simClustersMinScore = params(RelatedTweetProducerBasedParams.SimClustersMinScoreParam) // SimClusters - Experimental SANN Similarity Engine val enableExperimentalSimClustersANN = params( RelatedTweetProducerBasedParams.EnableExperimentalSimClustersANNParam) val experimentalSimClustersANNConfigId = params( SimClustersANNParams.ExperimentalSimClustersANNConfigId) // SimClusters - SANN cluster 1 Similarity Engine val enableSimClustersANN1 = params(RelatedTweetProducerBasedParams.EnableSimClustersANN1Param) val simClustersANN1ConfigId = params(SimClustersANNParams.SimClustersANN1ConfigId) // SimClusters - SANN cluster 2 Similarity Engine val enableSimClustersANN2 = params(RelatedTweetProducerBasedParams.EnableSimClustersANN2Param) val simClustersANN2ConfigId = params(SimClustersANNParams.SimClustersANN2ConfigId) // SimClusters - SANN cluster 3 Similarity Engine val enableSimClustersANN3 = params(RelatedTweetProducerBasedParams.EnableSimClustersANN3Param) val simClustersANN3ConfigId = params(SimClustersANNParams.SimClustersANN3ConfigId) // SimClusters - SANN cluster 5 Similarity Engine val enableSimClustersANN5 = params(RelatedTweetProducerBasedParams.EnableSimClustersANN5Param) val simClustersANN5ConfigId = params(SimClustersANNParams.SimClustersANN5ConfigId) val enableSimClustersANN4 = params(RelatedTweetProducerBasedParams.EnableSimClustersANN4Param) val simClustersANN4ConfigId = params(SimClustersANNParams.SimClustersANN4ConfigId) // Build SANN Query val simClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.FavBasedProducer, simClustersModelVersion, simClustersANNConfigId, params ) val experimentalSimClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.FavBasedProducer, simClustersModelVersion, experimentalSimClustersANNConfigId, params ) val simClustersANN1Query = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.FavBasedProducer, simClustersModelVersion, simClustersANN1ConfigId, params ) val simClustersANN2Query = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.FavBasedProducer, simClustersModelVersion, simClustersANN2ConfigId, params ) val simClustersANN3Query = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.FavBasedProducer, simClustersModelVersion, simClustersANN3ConfigId, params ) val simClustersANN5Query = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.FavBasedProducer, simClustersModelVersion, simClustersANN5ConfigId, params ) val simClustersANN4Query = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.FavBasedProducer, simClustersModelVersion, simClustersANN4ConfigId, params ) // UTG val enableUtg = params(RelatedTweetProducerBasedParams.EnableUTGParam) val utgCombinationMethod = params( ProducerBasedCandidateGenerationParams.UtgCombinationMethodParam) // SourceType.RequestUserId is a placeholder. val sourceInfo = SourceInfo(SourceType.RequestUserId, internalId, None) EngineQuery( Query( sourceInfo = sourceInfo, maxCandidateNumPerSourceKey = maxCandidateNumPerSourceKey, maxTweetAgeHours = maxTweetAgeHours, enableSimClustersANN = enableSimClustersANN, simClustersANNQuery = simClustersANNQuery, enableExperimentalSimClustersANN = enableExperimentalSimClustersANN, experimentalSimClustersANNQuery = experimentalSimClustersANNQuery, enableSimClustersANN1 = enableSimClustersANN1, simClustersANN1Query = simClustersANN1Query, enableSimClustersANN2 = enableSimClustersANN2, simClustersANN2Query = simClustersANN2Query, enableSimClustersANN3 = enableSimClustersANN3, simClustersANN3Query = simClustersANN3Query, enableSimClustersANN5 = enableSimClustersANN5, simClustersANN5Query = simClustersANN5Query, enableSimClustersANN4 = enableSimClustersANN4, simClustersANN4Query = simClustersANN4Query, simClustersMinScore = simClustersMinScore, enableUtg = enableUtg, utgQuery = ProducerBasedUserTweetGraphSimilarityEngine.fromParams(internalId, params), utgCombinationMethod = utgCombinationMethod ), params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUserAdGraphSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.param.ProducerBasedUserAdGraphParams import com.twitter.finagle.stats.StatsReceiver import com.twitter.recos.user_ad_graph.thriftscala.ProducerBasedRelatedAdRequest import com.twitter.recos.user_ad_graph.thriftscala.UserAdGraph import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import javax.inject.Singleton import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.frigate.common.util.StatsUtil import com.twitter.timelines.configapi /** * This store looks for similar tweets from UserAdGraph for a Source ProducerId * For a query producerId,User Tweet Graph (UAG), * lets us find out which ad tweets the query producer's followers co-engaged */ @Singleton case class ProducerBasedUserAdGraphSimilarityEngine( userAdGraphService: UserAdGraph.MethodPerEndpoint, statsReceiver: StatsReceiver) extends ReadableStore[ProducerBasedUserAdGraphSimilarityEngine.Query, Seq[ TweetWithScore ]] { private val stats = statsReceiver.scope(this.getClass.getSimpleName) private val fetchCandidatesStat = stats.scope("fetchCandidates") override def get( query: ProducerBasedUserAdGraphSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { query.sourceId match { case InternalId.UserId(producerId) => StatsUtil.trackOptionItemsStats(fetchCandidatesStat) { val relatedAdRequest = ProducerBasedRelatedAdRequest( producerId, maxResults = Some(query.maxResults), minCooccurrence = Some(query.minCooccurrence), minScore = Some(query.minScore), maxNumFollowers = Some(query.maxNumFollowers), maxTweetAgeInHours = Some(query.maxTweetAgeInHours), ) userAdGraphService.producerBasedRelatedAds(relatedAdRequest).map { relatedAdResponse => val candidates = relatedAdResponse.adTweets.map(tweet => TweetWithScore(tweet.adTweetId, tweet.score)) Some(candidates) } } case _ => Future.value(None) } } } object ProducerBasedUserAdGraphSimilarityEngine { def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo = { SimilarityEngineInfo( similarityEngineType = SimilarityEngineType.ProducerBasedUserAdGraph, modelId = None, score = Some(score)) } case class Query( sourceId: InternalId, maxResults: Int, minCooccurrence: Int, // require at least {minCooccurrence} lhs user engaged with returned tweet minScore: Double, maxNumFollowers: Int, // max number of lhs users maxTweetAgeInHours: Int) def fromParams( sourceId: InternalId, params: configapi.Params, ): EngineQuery[Query] = { EngineQuery( Query( sourceId = sourceId, maxResults = params(GlobalParams.MaxCandidateNumPerSourceKeyParam), minCooccurrence = params(ProducerBasedUserAdGraphParams.MinCoOccurrenceParam), maxNumFollowers = params(ProducerBasedUserAdGraphParams.MaxNumFollowersParam), maxTweetAgeInHours = params(GlobalParams.MaxTweetAgeHoursParam).inHours, minScore = params(ProducerBasedUserAdGraphParams.MinScoreParam) ), params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUserTweetGraphSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.ProducerBasedUserTweetGraphParams import com.twitter.finagle.stats.StatsReceiver import com.twitter.recos.user_tweet_graph.thriftscala.ProducerBasedRelatedTweetRequest import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import javax.inject.Singleton import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.frigate.common.util.StatsUtil import com.twitter.timelines.configapi import com.twitter.recos.user_tweet_graph.thriftscala.UserTweetGraph /** * This store looks for similar tweets from UserTweetGraph for a Source ProducerId * For a query producerId,User Tweet Graph (UTG), * lets us find out which tweets the query producer's followers co-engaged */ @Singleton case class ProducerBasedUserTweetGraphSimilarityEngine( userTweetGraphService: UserTweetGraph.MethodPerEndpoint, statsReceiver: StatsReceiver) extends ReadableStore[ProducerBasedUserTweetGraphSimilarityEngine.Query, Seq[ TweetWithScore ]] { private val stats = statsReceiver.scope(this.getClass.getSimpleName) private val fetchCandidatesStat = stats.scope("fetchCandidates") override def get( query: ProducerBasedUserTweetGraphSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { query.sourceId match { case InternalId.UserId(producerId) => StatsUtil.trackOptionItemsStats(fetchCandidatesStat) { val relatedTweetRequest = ProducerBasedRelatedTweetRequest( producerId, maxResults = Some(query.maxResults), minCooccurrence = Some(query.minCooccurrence), minScore = Some(query.minScore), maxNumFollowers = Some(query.maxNumFollowers), maxTweetAgeInHours = Some(query.maxTweetAgeInHours), ) userTweetGraphService.producerBasedRelatedTweets(relatedTweetRequest).map { relatedTweetResponse => val candidates = relatedTweetResponse.tweets.map(tweet => TweetWithScore(tweet.tweetId, tweet.score)) Some(candidates) } } case _ => Future.value(None) } } } object ProducerBasedUserTweetGraphSimilarityEngine { def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo = { SimilarityEngineInfo( similarityEngineType = SimilarityEngineType.ProducerBasedUserTweetGraph, modelId = None, score = Some(score)) } case class Query( sourceId: InternalId, maxResults: Int, minCooccurrence: Int, // require at least {minCooccurrence} lhs user engaged with returned tweet minScore: Double, maxNumFollowers: Int, // max number of lhs users maxTweetAgeInHours: Int) def fromParams( sourceId: InternalId, params: configapi.Params, ): EngineQuery[Query] = { EngineQuery( Query( sourceId = sourceId, maxResults = params(GlobalParams.MaxCandidateNumPerSourceKeyParam), minCooccurrence = params(ProducerBasedUserTweetGraphParams.MinCoOccurrenceParam), maxNumFollowers = params(ProducerBasedUserTweetGraphParams.MaxNumFollowersParam), maxTweetAgeInHours = params(GlobalParams.MaxTweetAgeHoursParam).inHours, minScore = params(ProducerBasedUserTweetGraphParams.MinScoreParam) ), params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimClustersANNSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.config.SimClustersANNConfig import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.simclusters_v2.thriftscala.EmbeddingType import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.simclusters_v2.thriftscala.ModelVersion import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId import com.twitter.simclustersann.thriftscala.SimClustersANNService import com.twitter.simclustersann.thriftscala.{Query => SimClustersANNQuery} import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.util.Future import javax.inject.Singleton import com.twitter.cr_mixer.exception.InvalidSANNConfigException import com.twitter.relevance_platform.simclustersann.multicluster.ServiceNameMapper @Singleton case class SimClustersANNSimilarityEngine( simClustersANNServiceNameToClientMapper: Map[String, SimClustersANNService.MethodPerEndpoint], statsReceiver: StatsReceiver) extends ReadableStore[ SimClustersANNSimilarityEngine.Query, Seq[TweetWithScore] ] { private val name: String = this.getClass.getSimpleName private val stats = statsReceiver.scope(name) private val fetchCandidatesStat = stats.scope("fetchCandidates") private def getSimClustersANNService( query: SimClustersANNQuery ): Option[SimClustersANNService.MethodPerEndpoint] = { ServiceNameMapper .getServiceName( query.sourceEmbeddingId.modelVersion, query.config.candidateEmbeddingType).flatMap(serviceName => simClustersANNServiceNameToClientMapper.get(serviceName)) } override def get( query: SimClustersANNSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { StatsUtil.trackOptionItemsStats(fetchCandidatesStat) { getSimClustersANNService(query.simClustersANNQuery) match { case Some(simClustersANNService) => simClustersANNService.getTweetCandidates(query.simClustersANNQuery).map { simClustersANNTweetCandidates => val tweetWithScores = simClustersANNTweetCandidates.map { candidate => TweetWithScore(candidate.tweetId, candidate.score) } Some(tweetWithScores) } case None => throw InvalidSANNConfigException( "No SANN Cluster configured to serve this query, check CandidateEmbeddingType and ModelVersion") } } } } object SimClustersANNSimilarityEngine { case class Query( simClustersANNQuery: SimClustersANNQuery, simClustersANNConfigId: String) def toSimilarityEngineInfo( query: EngineQuery[Query], score: Double ): SimilarityEngineInfo = { SimilarityEngineInfo( similarityEngineType = SimilarityEngineType.SimClustersANN, modelId = Some( s"SimClustersANN_${query.storeQuery.simClustersANNQuery.sourceEmbeddingId.embeddingType.toString}_" + s"${query.storeQuery.simClustersANNQuery.sourceEmbeddingId.modelVersion.toString}_" + s"${query.storeQuery.simClustersANNConfigId}"), score = Some(score) ) } def fromParams( internalId: InternalId, embeddingType: EmbeddingType, modelVersion: ModelVersion, simClustersANNConfigId: String, params: configapi.Params, ): EngineQuery[Query] = { // SimClusters EmbeddingId and ANNConfig val simClustersEmbeddingId = SimClustersEmbeddingId(embeddingType, modelVersion, internalId) val simClustersANNConfig = SimClustersANNConfig .getConfig(embeddingType.toString, modelVersion.toString, simClustersANNConfigId) EngineQuery( Query( SimClustersANNQuery( sourceEmbeddingId = simClustersEmbeddingId, config = simClustersANNConfig.toSANNConfigThrift ), simClustersANNConfigId ), params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.GlobalRequestTimeoutException import com.twitter.finagle.mux.ClientDiscardedRequestException import com.twitter.finagle.memcached.Client import com.twitter.finagle.mux.ServerApplicationError import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.hashing.KeyHasher import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.relevance_platform.common.injection.LZ4Injection import com.twitter.relevance_platform.common.injection.SeqObjectInjection import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.Params import com.twitter.util.Duration import com.twitter.util.Future import com.twitter.util.TimeoutException import com.twitter.util.logging.Logging import org.apache.thrift.TApplicationException /** * A SimilarityEngine is a wrapper which, given a [[Query]], returns a list of [[Candidate]] * The main purposes of a SimilarityEngine is to provide a consistent interface for candidate * generation logic, and provides default functions, including: * - Identification * - Observability * - Timeout settings * - Exception Handling * - Gating by Deciders & FeatureSwitch settings * - (coming soon): Dark traffic * * Note: * A SimilarityEngine by itself is NOT meant to be cacheable. * Caching should be implemented in the underlying ReadableStore that provides the [[Candidate]]s * * Please keep extension of this class local this directory only * */ trait SimilarityEngine[Query, Candidate] { /** * Uniquely identifies a similarity engine. * Avoid using the same engine type for more than one engine, it will cause stats to double count */ private[similarity_engine] def identifier: SimilarityEngineType def getCandidates(query: Query): Future[Option[Seq[Candidate]]] } object SimilarityEngine extends Logging { case class SimilarityEngineConfig( timeout: Duration, gatingConfig: GatingConfig) /** * Controls for whether or not this Engine is enabled. * In our previous design, we were expecting a Sim Engine will only take one set of Params, * and that’s why we decided to have GatingConfig and the EnableFeatureSwitch in the trait. * However, we now have two candidate generation pipelines: Tweet Rec, Related Tweets * and they are now having their own set of Params, but EnableFeatureSwitch can only put in 1 fixed value. * We need some further refactor work to make it more flexible. * * @param deciderConfig Gate the Engine by a decider. If specified, * @param enableFeatureSwitch. DO NOT USE IT FOR NOW. It needs some refactorting. Please set it to None (SD-20268) */ case class GatingConfig( deciderConfig: Option[DeciderConfig], enableFeatureSwitch: Option[ FSParam[Boolean] ]) // Do NOT use the enableFeatureSwitch. It needs some refactoring. case class DeciderConfig( decider: CrMixerDecider, deciderString: String) case class MemCacheConfig[K]( cacheClient: Client, ttl: Duration, asyncUpdate: Boolean = false, keyToString: K => String) private[similarity_engine] def isEnabled( params: Params, gatingConfig: GatingConfig ): Boolean = { val enabledByDecider = gatingConfig.deciderConfig.forall { config => config.decider.isAvailable(config.deciderString) } val enabledByFS = gatingConfig.enableFeatureSwitch.forall(params.apply) enabledByDecider && enabledByFS } // Default key hasher for memcache keys val keyHasher: KeyHasher = KeyHasher.FNV1A_64 /** * Add a MemCache wrapper to a ReadableStore with a preset key and value injection functions * Note: The [[Query]] object needs to be cacheable, * i.e. it cannot be a runtime objects or complex objects, for example, configapi.Params * * @param underlyingStore un-cached store implementation * @param keyPrefix a prefix differentiates 2 stores if they share the same key space. * e.x. 2 implementations of ReadableStore[UserId, Seq[Candidiate] ] * can use prefix "store_v1", "store_v2" * @return A ReadableStore with a MemCache wrapper */ private[similarity_engine] def addMemCache[Query, Candidate <: Serializable]( underlyingStore: ReadableStore[Query, Seq[Candidate]], memCacheConfig: MemCacheConfig[Query], keyPrefix: Option[String] = None, statsReceiver: StatsReceiver ): ReadableStore[Query, Seq[Candidate]] = { val prefix = keyPrefix.getOrElse("") ObservedMemcachedReadableStore.fromCacheClient[Query, Seq[Candidate]]( backingStore = underlyingStore, cacheClient = memCacheConfig.cacheClient, ttl = memCacheConfig.ttl, asyncUpdate = memCacheConfig.asyncUpdate, )( valueInjection = LZ4Injection.compose(SeqObjectInjection[Candidate]()), keyToString = { k: Query => s"CRMixer:$prefix${memCacheConfig.keyToString(k)}" }, statsReceiver = statsReceiver ) } private val timer = com.twitter.finagle.util.DefaultTimer /** * Applies runtime configs, like stats, timeouts, exception handling, onto fn */ private[similarity_engine] def getFromFn[Query, Candidate]( fn: Query => Future[Option[Seq[Candidate]]], storeQuery: Query, engineConfig: SimilarityEngineConfig, params: Params, scopedStats: StatsReceiver ): Future[Option[Seq[Candidate]]] = { if (isEnabled(params, engineConfig.gatingConfig)) { scopedStats.counter("gate_enabled").incr() StatsUtil .trackOptionItemsStats(scopedStats) { fn.apply(storeQuery).raiseWithin(engineConfig.timeout)(timer) } .rescue { case _: TimeoutException | _: GlobalRequestTimeoutException | _: TApplicationException | _: ClientDiscardedRequestException | _: ServerApplicationError // TApplicationException inside => debug("Failed to fetch. request aborted or timed out") Future.None case e => error("Failed to fetch. request aborted or timed out", e) Future.None } } else { scopedStats.counter("gate_disabled").incr() Future.None } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimilaritySourceOrderingUtil.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.TweetWithCandidateGenerationInfo import com.twitter.simclusters_v2.common.TweetId import scala.collection.mutable import scala.collection.mutable.ArrayBuffer object SimilaritySourceOrderingUtil { /** * This function flatten and dedup input candidates according to the order in the input Seq * [[candidate10, candidate11], [candidate20, candidate21]] => [candidate10, candidate11, candidate20, candidate21] */ def keepGivenOrder( candidates: Seq[Seq[TweetWithCandidateGenerationInfo]], ): Seq[TweetWithCandidateGenerationInfo] = { val seen = mutable.Set[TweetId]() val combinedCandidates = candidates.flatten val result = ArrayBuffer[TweetWithCandidateGenerationInfo]() combinedCandidates.foreach { candidate => val candidateTweetId = candidate.tweetId val seenCandidate = seen.contains(candidateTweetId) // de-dup if (!seenCandidate) { result += candidate seen.add(candidate.tweetId) } } //convert result to immutable seq result.toList } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SkitHighPrecisionTopicTweetSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.google.inject.Inject import com.google.inject.Singleton import com.google.inject.name.Named import com.twitter.contentrecommender.thriftscala.AlgorithmType import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TopicTweetWithScore import com.twitter.cr_mixer.param.TopicTweetParams import com.twitter.cr_mixer.similarity_engine.SkitTopicTweetSimilarityEngine._ import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.simclusters_v2.thriftscala.EmbeddingType import com.twitter.simclusters_v2.thriftscala.TopicId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.topic_recos.thriftscala.TopicTweet import com.twitter.topic_recos.thriftscala.TopicTweetPartitionFlatKey import com.twitter.util.Future @Singleton case class SkitHighPrecisionTopicTweetSimilarityEngine @Inject() ( @Named(ModuleNames.SkitStratoStoreName) skitStratoStore: ReadableStore[ TopicTweetPartitionFlatKey, Seq[TopicTweet] ], statsReceiver: StatsReceiver) extends ReadableStore[EngineQuery[Query], Seq[TopicTweetWithScore]] { private val name: String = this.getClass.getSimpleName private val stats = statsReceiver.scope(name) override def get(query: EngineQuery[Query]): Future[Option[Seq[TopicTweetWithScore]]] = { StatsUtil.trackOptionItemsStats(stats) { fetch(query).map { tweets => val topTweets = tweets .sortBy(-_.favCount) .take(query.storeQuery.maxCandidates) .map { tweet => TopicTweetWithScore( tweetId = tweet.tweetId, score = tweet.favCount, similarityEngineType = SimilarityEngineType.SkitHighPrecisionTopicTweet ) } Some(topTweets) } } } private def fetch(query: EngineQuery[Query]): Future[Seq[SkitTopicTweet]] = { val latestTweetTimeInHour = System.currentTimeMillis() / 1000 / 60 / 60 val earliestTweetTimeInHour = latestTweetTimeInHour - math.min(MaxTweetAgeInHours, query.storeQuery.maxTweetAge.inHours) val timedKeys = for (timePartition <- earliestTweetTimeInHour to latestTweetTimeInHour) yield { TopicTweetPartitionFlatKey( entityId = query.storeQuery.topicId.entityId, timePartition = timePartition, algorithmType = Some(AlgorithmType.SemanticCoreTweet), tweetEmbeddingType = Some(EmbeddingType.LogFavBasedTweet), language = query.storeQuery.topicId.language.getOrElse("").toLowerCase, country = None, // Disable country. It is not used. semanticCoreAnnotationVersionId = Some(query.storeQuery.semanticCoreVersionId) ) } getTweetsForKeys( timedKeys, query.storeQuery.topicId ) } /** * Given a set of keys, multiget the underlying Strato store, combine and flatten the results. */ private def getTweetsForKeys( keys: Seq[TopicTweetPartitionFlatKey], sourceTopic: TopicId ): Future[Seq[SkitTopicTweet]] = { Future .collect { skitStratoStore.multiGet(keys.toSet).values.toSeq } .map { combinedResults => val topTweets = combinedResults.flatten.flatten topTweets.map { tweet => SkitTopicTweet( tweetId = tweet.tweetId, favCount = tweet.scores.favCount.getOrElse(0L), cosineSimilarityScore = tweet.scores.cosineSimilarity.getOrElse(0.0), sourceTopic = sourceTopic ) } } } } object SkitHighPrecisionTopicTweetSimilarityEngine { def fromParams( topicId: TopicId, isVideoOnly: Boolean, params: configapi.Params, ): EngineQuery[Query] = { val maxCandidates = if (isVideoOnly) { params(TopicTweetParams.MaxSkitHighPrecisionCandidatesParam) * 2 } else { params(TopicTweetParams.MaxSkitHighPrecisionCandidatesParam) } EngineQuery( Query( topicId = topicId, maxCandidates = maxCandidates, maxTweetAge = params(TopicTweetParams.MaxTweetAge), semanticCoreVersionId = params(TopicTweetParams.SemanticCoreVersionIdParam) ), params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SkitTopicTweetSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.google.inject.Inject import com.google.inject.Singleton import com.google.inject.name.Named import com.twitter.contentrecommender.thriftscala.AlgorithmType import com.twitter.conversions.DurationOps._ import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.TopicTweetWithScore import com.twitter.cr_mixer.param.TopicTweetParams import com.twitter.cr_mixer.similarity_engine.SkitTopicTweetSimilarityEngine._ import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.thriftscala.EmbeddingType import com.twitter.simclusters_v2.thriftscala.ModelVersion import com.twitter.simclusters_v2.thriftscala.TopicId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.topic_recos.thriftscala.TopicTweet import com.twitter.topic_recos.thriftscala.TopicTweetPartitionFlatKey import com.twitter.util.Duration import com.twitter.util.Future @Singleton case class SkitTopicTweetSimilarityEngine @Inject() ( @Named(ModuleNames.SkitStratoStoreName) skitStratoStore: ReadableStore[ TopicTweetPartitionFlatKey, Seq[TopicTweet] ], statsReceiver: StatsReceiver) extends ReadableStore[EngineQuery[Query], Seq[TopicTweetWithScore]] { private val name: String = this.getClass.getSimpleName private val stats = statsReceiver.scope(name) override def get(query: EngineQuery[Query]): Future[Option[Seq[TopicTweetWithScore]]] = { StatsUtil.trackOptionItemsStats(stats) { fetch(query).map { tweets => val topTweets = tweets .sortBy(-_.cosineSimilarityScore) .take(query.storeQuery.maxCandidates) .map { tweet => TopicTweetWithScore( tweetId = tweet.tweetId, score = tweet.cosineSimilarityScore, similarityEngineType = SimilarityEngineType.SkitTfgTopicTweet ) } Some(topTweets) } } } private def fetch(query: EngineQuery[Query]): Future[Seq[SkitTopicTweet]] = { val latestTweetTimeInHour = System.currentTimeMillis() / 1000 / 60 / 60 val earliestTweetTimeInHour = latestTweetTimeInHour - math.min(MaxTweetAgeInHours, query.storeQuery.maxTweetAge.inHours) val timedKeys = for (timePartition <- earliestTweetTimeInHour to latestTweetTimeInHour) yield { TopicTweetPartitionFlatKey( entityId = query.storeQuery.topicId.entityId, timePartition = timePartition, algorithmType = Some(AlgorithmType.TfgTweet), tweetEmbeddingType = Some(EmbeddingType.LogFavBasedTweet), language = query.storeQuery.topicId.language.getOrElse("").toLowerCase, country = None, // Disable country. It is not used. semanticCoreAnnotationVersionId = Some(query.storeQuery.semanticCoreVersionId), simclustersModelVersion = Some(ModelVersion.Model20m145k2020) ) } getTweetsForKeys( timedKeys, query.storeQuery.topicId ) } /** * Given a set of keys, multiget the underlying Strato store, combine and flatten the results. */ private def getTweetsForKeys( keys: Seq[TopicTweetPartitionFlatKey], sourceTopic: TopicId ): Future[Seq[SkitTopicTweet]] = { Future .collect { skitStratoStore.multiGet(keys.toSet).values.toSeq } .map { combinedResults => val topTweets = combinedResults.flatten.flatten topTweets.map { tweet => SkitTopicTweet( tweetId = tweet.tweetId, favCount = tweet.scores.favCount.getOrElse(0L), cosineSimilarityScore = tweet.scores.cosineSimilarity.getOrElse(0.0), sourceTopic = sourceTopic ) } } } } object SkitTopicTweetSimilarityEngine { val MaxTweetAgeInHours: Int = 7.days.inHours // Simple guard to prevent overloading // Query is used as a cache key. Do not add any user level information in this. case class Query( topicId: TopicId, maxCandidates: Int, maxTweetAge: Duration, semanticCoreVersionId: Long) case class SkitTopicTweet( sourceTopic: TopicId, tweetId: TweetId, favCount: Long, cosineSimilarityScore: Double) def fromParams( topicId: TopicId, isVideoOnly: Boolean, params: configapi.Params, ): EngineQuery[Query] = { val maxCandidates = if (isVideoOnly) { params(TopicTweetParams.MaxSkitTfgCandidatesParam) * 2 } else { params(TopicTweetParams.MaxSkitTfgCandidatesParam) } EngineQuery( Query( topicId = topicId, maxCandidates = maxCandidates, maxTweetAge = params(TopicTweetParams.MaxTweetAge), semanticCoreVersionId = params(TopicTweetParams.SemanticCoreVersionIdParam) ), params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/StandardSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.MemCacheConfig import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi.Params import com.twitter.util.Future /** * @tparam Query ReadableStore's input type. */ case class EngineQuery[Query]( storeQuery: Query, params: Params, ) /** * A straight forward SimilarityEngine implementation that wraps a ReadableStore * * @param implementingStore Provides the candidate retrieval's implementations * @param memCacheConfig If specified, it will wrap the underlying store with a MemCache layer * You should only enable this for cacheable queries, e.x. TweetIds. * consumer based UserIds are generally not possible to cache. * @tparam Query ReadableStore's input type * @tparam Candidate ReadableStore's return type is Seq[[[Candidate]]] */ class StandardSimilarityEngine[Query, Candidate <: Serializable]( implementingStore: ReadableStore[Query, Seq[Candidate]], override val identifier: SimilarityEngineType, globalStats: StatsReceiver, engineConfig: SimilarityEngineConfig, memCacheConfig: Option[MemCacheConfig[Query]] = None) extends SimilarityEngine[EngineQuery[Query], Candidate] { private val scopedStats = globalStats.scope("similarityEngine", identifier.toString) def getScopedStats: StatsReceiver = scopedStats // Add memcache wrapper, if specified private val store = { memCacheConfig match { case Some(config) => SimilarityEngine.addMemCache( underlyingStore = implementingStore, memCacheConfig = config, statsReceiver = scopedStats ) case _ => implementingStore } } override def getCandidates( engineQuery: EngineQuery[Query] ): Future[Option[Seq[Candidate]]] = { SimilarityEngine.getFromFn( store.get, engineQuery.storeQuery, engineConfig, engineQuery.params, scopedStats ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedQigSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.base.Stats import com.twitter.product_mixer.core.thriftscala.ClientContext import com.twitter.qig_ranker.thriftscala.Product import com.twitter.qig_ranker.thriftscala.ProductContext import com.twitter.qig_ranker.thriftscala.QigRanker import com.twitter.qig_ranker.thriftscala.QigRankerProductResponse import com.twitter.qig_ranker.thriftscala.QigRankerRequest import com.twitter.qig_ranker.thriftscala.QigRankerResponse import com.twitter.qig_ranker.thriftscala.TwistlySimilarTweetsProductContext import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.util.Future import javax.inject.Singleton /** * This store looks for similar tweets from QueryInteractionGraph (QIG) for a source tweet id. * For a given query tweet, QIG returns us the similar tweets that have an overlap of engagements * (with the query tweet) on different search queries */ @Singleton case class TweetBasedQigSimilarityEngine( qigRanker: QigRanker.MethodPerEndpoint, statsReceiver: StatsReceiver) extends ReadableStore[ TweetBasedQigSimilarityEngine.Query, Seq[TweetWithScore] ] { private val stats = statsReceiver.scope(this.getClass.getSimpleName) private val fetchCandidatesStat = stats.scope("fetchCandidates") override def get( query: TweetBasedQigSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { query.sourceId match { case InternalId.TweetId(tweetId) => val qigSimilarTweetsRequest = getQigSimilarTweetsRequest(tweetId) Stats.trackOption(fetchCandidatesStat) { qigRanker .getSimilarCandidates(qigSimilarTweetsRequest) .map { qigSimilarTweetsResponse => getCandidatesFromQigResponse(qigSimilarTweetsResponse) } } case _ => Future.value(None) } } private def getQigSimilarTweetsRequest( tweetId: Long ): QigRankerRequest = { // Note: QigRanker needs a non-empty userId to be passed to return results. // We are passing in a dummy userId until we fix this on QigRanker side val clientContext = ClientContext(userId = Some(0L)) val productContext = ProductContext.TwistlySimilarTweetsProductContext( TwistlySimilarTweetsProductContext(tweetId = tweetId)) QigRankerRequest( clientContext = clientContext, product = Product.TwistlySimilarTweets, productContext = Some(productContext), ) } private def getCandidatesFromQigResponse( qigSimilarTweetsResponse: QigRankerResponse ): Option[Seq[TweetWithScore]] = { qigSimilarTweetsResponse.productResponse match { case QigRankerProductResponse .TwistlySimilarTweetCandidatesResponse(response) => val tweetsWithScore = response.similarTweets .map { similarTweetResult => TweetWithScore( similarTweetResult.tweetResult.tweetId, similarTweetResult.tweetResult.score.getOrElse(0L)) } Some(tweetsWithScore) case _ => None } } } object TweetBasedQigSimilarityEngine { def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo = { SimilarityEngineInfo( similarityEngineType = SimilarityEngineType.Qig, modelId = None, score = Some(score)) } case class Query(sourceId: InternalId) def fromParams( sourceId: InternalId, params: configapi.Params, ): EngineQuery[Query] = { EngineQuery( Query(sourceId = sourceId), params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUnifiedSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.CandidateGenerationInfo import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.model.TweetWithCandidateGenerationInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.param.RelatedTweetTweetBasedParams import com.twitter.cr_mixer.param.RelatedVideoTweetTweetBasedParams import com.twitter.cr_mixer.param.SimClustersANNParams import com.twitter.cr_mixer.param.TweetBasedCandidateGenerationParams import com.twitter.cr_mixer.param.TweetBasedTwHINParams import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.cr_mixer.util.InterleaveUtil import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.simclusters_v2.common.ModelVersions import com.twitter.simclusters_v2.thriftscala.EmbeddingType import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.snowflake.id.SnowflakeId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.util.Duration import com.twitter.util.Future import com.twitter.util.Time import javax.inject.Named import javax.inject.Singleton import scala.collection.mutable.ArrayBuffer /** * This store fetches similar tweets from multiple tweet based candidate sources * and combines them using different methods obtained from query params */ @Singleton case class TweetBasedUnifiedSimilarityEngine( @Named(ModuleNames.TweetBasedUserTweetGraphSimilarityEngine) tweetBasedUserTweetGraphSimilarityEngine: StandardSimilarityEngine[ TweetBasedUserTweetGraphSimilarityEngine.Query, TweetWithScore ], @Named(ModuleNames.TweetBasedUserVideoGraphSimilarityEngine) tweetBasedUserVideoGraphSimilarityEngine: StandardSimilarityEngine[ TweetBasedUserVideoGraphSimilarityEngine.Query, TweetWithScore ], simClustersANNSimilarityEngine: StandardSimilarityEngine[ SimClustersANNSimilarityEngine.Query, TweetWithScore ], @Named(ModuleNames.TweetBasedQigSimilarityEngine) tweetBasedQigSimilarTweetsSimilarityEngine: StandardSimilarityEngine[ TweetBasedQigSimilarityEngine.Query, TweetWithScore ], @Named(ModuleNames.TweetBasedTwHINANNSimilarityEngine) tweetBasedTwHINANNSimilarityEngine: HnswANNSimilarityEngine, statsReceiver: StatsReceiver) extends ReadableStore[ TweetBasedUnifiedSimilarityEngine.Query, Seq[TweetWithCandidateGenerationInfo] ] { import TweetBasedUnifiedSimilarityEngine._ private val stats = statsReceiver.scope(this.getClass.getSimpleName) private val fetchCandidatesStat = stats.scope("fetchCandidates") override def get( query: Query ): Future[Option[Seq[TweetWithCandidateGenerationInfo]]] = { query.sourceInfo.internalId match { case _: InternalId.TweetId => StatsUtil.trackOptionItemsStats(fetchCandidatesStat) { val twhinQuery = HnswANNEngineQuery( sourceId = query.sourceInfo.internalId, modelId = query.twhinModelId, params = query.params) val utgCandidatesFut = if (query.enableUtg) tweetBasedUserTweetGraphSimilarityEngine.getCandidates(query.utgQuery) else Future.None val uvgCandidatesFut = if (query.enableUvg) tweetBasedUserVideoGraphSimilarityEngine.getCandidates(query.uvgQuery) else Future.None val sannCandidatesFut = if (query.enableSimClustersANN) { simClustersANNSimilarityEngine.getCandidates(query.simClustersANNQuery) } else Future.None val sann1CandidatesFut = if (query.enableSimClustersANN1) { simClustersANNSimilarityEngine.getCandidates(query.simClustersANN1Query) } else Future.None val sann2CandidatesFut = if (query.enableSimClustersANN2) { simClustersANNSimilarityEngine.getCandidates(query.simClustersANN2Query) } else Future.None val sann3CandidatesFut = if (query.enableSimClustersANN3) { simClustersANNSimilarityEngine.getCandidates(query.simClustersANN3Query) } else Future.None val sann5CandidatesFut = if (query.enableSimClustersANN5) { simClustersANNSimilarityEngine.getCandidates(query.simClustersANN5Query) } else Future.None val sann4CandidatesFut = if (query.enableSimClustersANN4) { simClustersANNSimilarityEngine.getCandidates(query.simClustersANN4Query) } else Future.None val experimentalSANNCandidatesFut = if (query.enableExperimentalSimClustersANN) { simClustersANNSimilarityEngine.getCandidates(query.experimentalSimClustersANNQuery) } else Future.None val qigCandidatesFut = if (query.enableQig) tweetBasedQigSimilarTweetsSimilarityEngine.getCandidates(query.qigQuery) else Future.None val twHINCandidateFut = if (query.enableTwHIN) { tweetBasedTwHINANNSimilarityEngine.getCandidates(twhinQuery) } else Future.None Future .join( utgCandidatesFut, sannCandidatesFut, sann1CandidatesFut, sann2CandidatesFut, sann3CandidatesFut, sann5CandidatesFut, sann4CandidatesFut, experimentalSANNCandidatesFut, qigCandidatesFut, twHINCandidateFut, uvgCandidatesFut ).map { case ( userTweetGraphCandidates, simClustersANNCandidates, simClustersANN1Candidates, simClustersANN2Candidates, simClustersANN3Candidates, simClustersANN5Candidates, simClustersANN4Candidates, experimentalSANNCandidates, qigSimilarTweetsCandidates, twhinCandidates, userVideoGraphCandidates) => val filteredUTGTweets = userTweetGraphFilter(userTweetGraphCandidates.toSeq.flatten) val filteredUVGTweets = userVideoGraphFilter(userVideoGraphCandidates.toSeq.flatten) val filteredSANNTweets = simClustersCandidateMinScoreFilter( simClustersANNCandidates.toSeq.flatten, query.simClustersMinScore, query.simClustersANNQuery.storeQuery.simClustersANNConfigId) val filteredSANN1Tweets = simClustersCandidateMinScoreFilter( simClustersANN1Candidates.toSeq.flatten, query.simClustersMinScore, query.simClustersANN1Query.storeQuery.simClustersANNConfigId) val filteredSANN2Tweets = simClustersCandidateMinScoreFilter( simClustersANN2Candidates.toSeq.flatten, query.simClustersMinScore, query.simClustersANN2Query.storeQuery.simClustersANNConfigId) val filteredSANN3Tweets = simClustersCandidateMinScoreFilter( simClustersANN3Candidates.toSeq.flatten, query.simClustersMinScore, query.simClustersANN3Query.storeQuery.simClustersANNConfigId) val filteredSANN4Tweets = simClustersCandidateMinScoreFilter( simClustersANN4Candidates.toSeq.flatten, query.simClustersMinScore, query.simClustersANN4Query.storeQuery.simClustersANNConfigId) val filteredSANN5Tweets = simClustersCandidateMinScoreFilter( simClustersANN5Candidates.toSeq.flatten, query.simClustersMinScore, query.simClustersANN5Query.storeQuery.simClustersANNConfigId) val filteredExperimentalSANNTweets = simClustersCandidateMinScoreFilter( experimentalSANNCandidates.toSeq.flatten, query.simClustersVideoBasedMinScore, query.experimentalSimClustersANNQuery.storeQuery.simClustersANNConfigId) val filteredQigTweets = qigSimilarTweetsFilter( qigSimilarTweetsCandidates.toSeq.flatten, query.qigMaxTweetAgeHours, query.qigMaxNumSimilarTweets ) val filteredTwHINTweets = twhinFilter( twhinCandidates.toSeq.flatten.sortBy(-_.score), query.twhinMaxTweetAgeHours, tweetBasedTwHINANNSimilarityEngine.getScopedStats ) val utgTweetsWithCGInfo = filteredUTGTweets.map { tweetWithScore => val similarityEngineInfo = TweetBasedUserTweetGraphSimilarityEngine .toSimilarityEngineInfo(tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val uvgTweetsWithCGInfo = filteredUVGTweets.map { tweetWithScore => val similarityEngineInfo = TweetBasedUserVideoGraphSimilarityEngine .toSimilarityEngineInfo(tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val sannTweetsWithCGInfo = filteredSANNTweets.map { tweetWithScore => val similarityEngineInfo = SimClustersANNSimilarityEngine .toSimilarityEngineInfo(query.simClustersANNQuery, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val sann1TweetsWithCGInfo = filteredSANN1Tweets.map { tweetWithScore => val similarityEngineInfo = SimClustersANNSimilarityEngine .toSimilarityEngineInfo(query.simClustersANN1Query, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val sann2TweetsWithCGInfo = filteredSANN2Tweets.map { tweetWithScore => val similarityEngineInfo = SimClustersANNSimilarityEngine .toSimilarityEngineInfo(query.simClustersANN2Query, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val sann3TweetsWithCGInfo = filteredSANN3Tweets.map { tweetWithScore => val similarityEngineInfo = SimClustersANNSimilarityEngine .toSimilarityEngineInfo(query.simClustersANN3Query, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val sann4TweetsWithCGInfo = filteredSANN4Tweets.map { tweetWithScore => val similarityEngineInfo = SimClustersANNSimilarityEngine .toSimilarityEngineInfo(query.simClustersANN4Query, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val sann5TweetsWithCGInfo = filteredSANN5Tweets.map { tweetWithScore => val similarityEngineInfo = SimClustersANNSimilarityEngine .toSimilarityEngineInfo(query.simClustersANN5Query, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val experimentalSANNTweetsWithCGInfo = filteredExperimentalSANNTweets.map { tweetWithScore => val similarityEngineInfo = SimClustersANNSimilarityEngine .toSimilarityEngineInfo( query.experimentalSimClustersANNQuery, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val qigTweetsWithCGInfo = filteredQigTweets.map { tweetWithScore => val similarityEngineInfo = TweetBasedQigSimilarityEngine .toSimilarityEngineInfo(tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val twHINTweetsWithCGInfo = filteredTwHINTweets.map { tweetWithScore => val similarityEngineInfo = tweetBasedTwHINANNSimilarityEngine .toSimilarityEngineInfo(twhinQuery, tweetWithScore.score) TweetWithCandidateGenerationInfo( tweetWithScore.tweetId, CandidateGenerationInfo( Some(query.sourceInfo), similarityEngineInfo, Seq(similarityEngineInfo) )) } val candidateSourcesToBeInterleaved = ArrayBuffer[Seq[TweetWithCandidateGenerationInfo]]( sannTweetsWithCGInfo, experimentalSANNTweetsWithCGInfo, sann1TweetsWithCGInfo, sann2TweetsWithCGInfo, sann3TweetsWithCGInfo, sann5TweetsWithCGInfo, sann4TweetsWithCGInfo, qigTweetsWithCGInfo, uvgTweetsWithCGInfo, utgTweetsWithCGInfo, twHINTweetsWithCGInfo ) val interleavedCandidates = InterleaveUtil.interleave(candidateSourcesToBeInterleaved) val unifiedCandidatesWithUnifiedCGInfo = interleavedCandidates.map { candidate => /*** * when a candidate was made by interleave/keepGivenOrder, * then we apply getTweetBasedUnifiedCGInfo() to override with the unified CGInfo * * we'll not have ALL SEs that generated the tweet * in contributingSE list for interleave. We only have the chosen SE available. */ TweetWithCandidateGenerationInfo( tweetId = candidate.tweetId, candidateGenerationInfo = getTweetBasedUnifiedCGInfo( candidate.candidateGenerationInfo.sourceInfoOpt, candidate.getSimilarityScore, candidate.candidateGenerationInfo.contributingSimilarityEngines ) // getSimilarityScore comes from either unifiedScore or single score ) } stats .stat("unified_candidate_size").add(unifiedCandidatesWithUnifiedCGInfo.size) val truncatedCandidates = unifiedCandidatesWithUnifiedCGInfo.take(query.maxCandidateNumPerSourceKey) stats.stat("truncatedCandidates_size").add(truncatedCandidates.size) Some(truncatedCandidates) } } case _ => stats.counter("sourceId_is_not_tweetId_cnt").incr() Future.None } } private def simClustersCandidateMinScoreFilter( simClustersAnnCandidates: Seq[TweetWithScore], simClustersMinScore: Double, simClustersANNConfigId: String ): Seq[TweetWithScore] = { val filteredCandidates = simClustersAnnCandidates .filter { candidate => candidate.score > simClustersMinScore } stats.stat(simClustersANNConfigId, "simClustersAnnCandidates_size").add(filteredCandidates.size) stats.counter(simClustersANNConfigId, "simClustersAnnRequests").incr() if (filteredCandidates.isEmpty) stats.counter(simClustersANNConfigId, "emptyFilteredSimClustersAnnCandidates").incr() filteredCandidates.map { candidate => TweetWithScore(candidate.tweetId, candidate.score) } } /** Returns a list of tweets that are generated less than `maxTweetAgeHours` hours ago */ private def tweetAgeFilter( candidates: Seq[TweetWithScore], maxTweetAgeHours: Duration ): Seq[TweetWithScore] = { // Tweet IDs are approximately chronological (see http://go/snowflake), // so we are building the earliest tweet id once // The per-candidate logic here then be candidate.tweetId > earliestPermittedTweetId, which is far cheaper. val earliestTweetId = SnowflakeId.firstIdFor(Time.now - maxTweetAgeHours) candidates.filter { candidate => candidate.tweetId >= earliestTweetId } } private def twhinFilter( twhinCandidates: Seq[TweetWithScore], twhinMaxTweetAgeHours: Duration, simEngineStats: StatsReceiver ): Seq[TweetWithScore] = { simEngineStats.stat("twhinCandidates_size").add(twhinCandidates.size) val candidates = twhinCandidates.map { candidate => TweetWithScore(candidate.tweetId, candidate.score) } val filteredCandidates = tweetAgeFilter(candidates, twhinMaxTweetAgeHours) simEngineStats.stat("filteredTwhinCandidates_size").add(filteredCandidates.size) if (filteredCandidates.isEmpty) simEngineStats.counter("emptyFilteredTwhinCandidates").incr() filteredCandidates } /** A no-op filter as UTG filtering already happens on UTG service side */ private def userTweetGraphFilter( userTweetGraphCandidates: Seq[TweetWithScore] ): Seq[TweetWithScore] = { val filteredCandidates = userTweetGraphCandidates stats.stat("userTweetGraphCandidates_size").add(userTweetGraphCandidates.size) if (filteredCandidates.isEmpty) stats.counter("emptyFilteredUserTweetGraphCandidates").incr() filteredCandidates.map { candidate => TweetWithScore(candidate.tweetId, candidate.score) } } /** A no-op filter as UVG filtering already happens on UVG service side */ private def userVideoGraphFilter( userVideoGraphCandidates: Seq[TweetWithScore] ): Seq[TweetWithScore] = { val filteredCandidates = userVideoGraphCandidates stats.stat("userVideoGraphCandidates_size").add(userVideoGraphCandidates.size) if (filteredCandidates.isEmpty) stats.counter("emptyFilteredUserVideoGraphCandidates").incr() filteredCandidates.map { candidate => TweetWithScore(candidate.tweetId, candidate.score) } } private def qigSimilarTweetsFilter( qigSimilarTweetsCandidates: Seq[TweetWithScore], qigMaxTweetAgeHours: Duration, qigMaxNumSimilarTweets: Int ): Seq[TweetWithScore] = { val ageFilteredCandidates = tweetAgeFilter(qigSimilarTweetsCandidates, qigMaxTweetAgeHours) stats.stat("ageFilteredQigSimilarTweetsCandidates_size").add(ageFilteredCandidates.size) val filteredCandidates = ageFilteredCandidates.take(qigMaxNumSimilarTweets) if (filteredCandidates.isEmpty) stats.counter("emptyFilteredQigSimilarTweetsCandidates").incr() filteredCandidates } /*** * Every candidate will have the CG Info with TweetBasedUnifiedSimilarityEngine * as they are generated by a composite of Similarity Engines. * Additionally, we store the contributing SEs (eg., SANN, UTG). */ private def getTweetBasedUnifiedCGInfo( sourceInfoOpt: Option[SourceInfo], unifiedScore: Double, contributingSimilarityEngines: Seq[SimilarityEngineInfo] ): CandidateGenerationInfo = { CandidateGenerationInfo( sourceInfoOpt, SimilarityEngineInfo( similarityEngineType = SimilarityEngineType.TweetBasedUnifiedSimilarityEngine, modelId = None, // We do not assign modelId for a unified similarity engine score = Some(unifiedScore) ), contributingSimilarityEngines ) } } object TweetBasedUnifiedSimilarityEngine { case class Query( sourceInfo: SourceInfo, maxCandidateNumPerSourceKey: Int, enableSimClustersANN: Boolean, simClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], enableExperimentalSimClustersANN: Boolean, experimentalSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], enableSimClustersANN1: Boolean, simClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query], enableSimClustersANN2: Boolean, simClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query], enableSimClustersANN3: Boolean, simClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query], enableSimClustersANN5: Boolean, simClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query], enableSimClustersANN4: Boolean, simClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query], simClustersMinScore: Double, simClustersVideoBasedMinScore: Double, twhinModelId: String, enableTwHIN: Boolean, twhinMaxTweetAgeHours: Duration, qigMaxTweetAgeHours: Duration, qigMaxNumSimilarTweets: Int, enableUtg: Boolean, utgQuery: EngineQuery[TweetBasedUserTweetGraphSimilarityEngine.Query], enableUvg: Boolean, uvgQuery: EngineQuery[TweetBasedUserVideoGraphSimilarityEngine.Query], enableQig: Boolean, qigQuery: EngineQuery[TweetBasedQigSimilarityEngine.Query], params: configapi.Params) def fromParams( sourceInfo: SourceInfo, params: configapi.Params, ): EngineQuery[Query] = { // SimClusters val enableSimClustersANN = params(TweetBasedCandidateGenerationParams.EnableSimClustersANNParam) val simClustersModelVersion = ModelVersions.Enum.enumToSimClustersModelVersionMap(params(GlobalParams.ModelVersionParam)) val simClustersMinScore = params(TweetBasedCandidateGenerationParams.SimClustersMinScoreParam) val simClustersVideoBasedMinScore = params( TweetBasedCandidateGenerationParams.SimClustersVideoBasedMinScoreParam) val simClustersANNConfigId = params(SimClustersANNParams.SimClustersANNConfigId) // SimClusters - Experimental SANN Similarity Engine (Video based SE) val enableExperimentalSimClustersANN = params(TweetBasedCandidateGenerationParams.EnableExperimentalSimClustersANNParam) val experimentalSimClustersANNConfigId = params( SimClustersANNParams.ExperimentalSimClustersANNConfigId) // SimClusters - SANN cluster 1 Similarity Engine val enableSimClustersANN1 = params(TweetBasedCandidateGenerationParams.EnableSimClustersANN1Param) val simClustersANN1ConfigId = params(SimClustersANNParams.SimClustersANN1ConfigId) // SimClusters - SANN cluster 2 Similarity Engine val enableSimClustersANN2 = params(TweetBasedCandidateGenerationParams.EnableSimClustersANN2Param) val simClustersANN2ConfigId = params(SimClustersANNParams.SimClustersANN2ConfigId) // SimClusters - SANN cluster 3 Similarity Engine val enableSimClustersANN3 = params(TweetBasedCandidateGenerationParams.EnableSimClustersANN3Param) val simClustersANN3ConfigId = params(SimClustersANNParams.SimClustersANN3ConfigId) // SimClusters - SANN cluster 5 Similarity Engine val enableSimClustersANN5 = params(TweetBasedCandidateGenerationParams.EnableSimClustersANN5Param) val simClustersANN5ConfigId = params(SimClustersANNParams.SimClustersANN5ConfigId) // SimClusters - SANN cluster 4 Similarity Engine val enableSimClustersANN4 = params(TweetBasedCandidateGenerationParams.EnableSimClustersANN4Param) val simClustersANN4ConfigId = params(SimClustersANNParams.SimClustersANN4ConfigId) // SimClusters ANN Queries for different SANN clusters val simClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( sourceInfo.internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANNConfigId, params ) val experimentalSimClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( sourceInfo.internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, experimentalSimClustersANNConfigId, params ) val simClustersANN1Query = SimClustersANNSimilarityEngine.fromParams( sourceInfo.internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANN1ConfigId, params ) val simClustersANN2Query = SimClustersANNSimilarityEngine.fromParams( sourceInfo.internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANN2ConfigId, params ) val simClustersANN3Query = SimClustersANNSimilarityEngine.fromParams( sourceInfo.internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANN3ConfigId, params ) val simClustersANN5Query = SimClustersANNSimilarityEngine.fromParams( sourceInfo.internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANN5ConfigId, params ) val simClustersANN4Query = SimClustersANNSimilarityEngine.fromParams( sourceInfo.internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANN4ConfigId, params ) // TweetBasedCandidateGeneration val maxCandidateNumPerSourceKey = params(GlobalParams.MaxCandidateNumPerSourceKeyParam) // TwHIN val twhinModelId = params(TweetBasedTwHINParams.ModelIdParam) val enableTwHIN = params(TweetBasedCandidateGenerationParams.EnableTwHINParam) val twhinMaxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam) // QIG val enableQig = params(TweetBasedCandidateGenerationParams.EnableQigSimilarTweetsParam) val qigMaxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam) val qigMaxNumSimilarTweets = params( TweetBasedCandidateGenerationParams.QigMaxNumSimilarTweetsParam) // UTG val enableUtg = params(TweetBasedCandidateGenerationParams.EnableUTGParam) // UVG val enableUvg = params(TweetBasedCandidateGenerationParams.EnableUVGParam) EngineQuery( Query( sourceInfo = sourceInfo, maxCandidateNumPerSourceKey = maxCandidateNumPerSourceKey, enableSimClustersANN = enableSimClustersANN, simClustersANNQuery = simClustersANNQuery, enableExperimentalSimClustersANN = enableExperimentalSimClustersANN, experimentalSimClustersANNQuery = experimentalSimClustersANNQuery, enableSimClustersANN1 = enableSimClustersANN1, simClustersANN1Query = simClustersANN1Query, enableSimClustersANN2 = enableSimClustersANN2, simClustersANN2Query = simClustersANN2Query, enableSimClustersANN3 = enableSimClustersANN3, simClustersANN3Query = simClustersANN3Query, enableSimClustersANN5 = enableSimClustersANN5, simClustersANN5Query = simClustersANN5Query, enableSimClustersANN4 = enableSimClustersANN4, simClustersANN4Query = simClustersANN4Query, simClustersMinScore = simClustersMinScore, simClustersVideoBasedMinScore = simClustersVideoBasedMinScore, twhinModelId = twhinModelId, enableTwHIN = enableTwHIN, twhinMaxTweetAgeHours = twhinMaxTweetAgeHours, qigMaxTweetAgeHours = qigMaxTweetAgeHours, qigMaxNumSimilarTweets = qigMaxNumSimilarTweets, enableUtg = enableUtg, utgQuery = TweetBasedUserTweetGraphSimilarityEngine .fromParams(sourceInfo.internalId, params), enableQig = enableQig, qigQuery = TweetBasedQigSimilarityEngine.fromParams(sourceInfo.internalId, params), enableUvg = enableUvg, uvgQuery = TweetBasedUserVideoGraphSimilarityEngine.fromParams(sourceInfo.internalId, params), params = params ), params ) } def fromParamsForRelatedTweet( internalId: InternalId, params: configapi.Params, ): EngineQuery[Query] = { // SimClusters val enableSimClustersANN = params(RelatedTweetTweetBasedParams.EnableSimClustersANNParam) val simClustersModelVersion = ModelVersions.Enum.enumToSimClustersModelVersionMap(params(GlobalParams.ModelVersionParam)) val simClustersMinScore = params(RelatedTweetTweetBasedParams.SimClustersMinScoreParam) val simClustersANNConfigId = params(SimClustersANNParams.SimClustersANNConfigId) val enableExperimentalSimClustersANN = params(RelatedTweetTweetBasedParams.EnableExperimentalSimClustersANNParam) val experimentalSimClustersANNConfigId = params( SimClustersANNParams.ExperimentalSimClustersANNConfigId) // SimClusters - SANN cluster 1 Similarity Engine val enableSimClustersANN1 = params(RelatedTweetTweetBasedParams.EnableSimClustersANN1Param) val simClustersANN1ConfigId = params(SimClustersANNParams.SimClustersANN1ConfigId) // SimClusters - SANN cluster 2 Similarity Engine val enableSimClustersANN2 = params(RelatedTweetTweetBasedParams.EnableSimClustersANN2Param) val simClustersANN2ConfigId = params(SimClustersANNParams.SimClustersANN2ConfigId) // SimClusters - SANN cluster 3 Similarity Engine val enableSimClustersANN3 = params(RelatedTweetTweetBasedParams.EnableSimClustersANN3Param) val simClustersANN3ConfigId = params(SimClustersANNParams.SimClustersANN3ConfigId) // SimClusters - SANN cluster 5 Similarity Engine val enableSimClustersANN5 = params(RelatedTweetTweetBasedParams.EnableSimClustersANN5Param) val simClustersANN5ConfigId = params(SimClustersANNParams.SimClustersANN5ConfigId) // SimClusters - SANN cluster 4 Similarity Engine val enableSimClustersANN4 = params(RelatedTweetTweetBasedParams.EnableSimClustersANN4Param) val simClustersANN4ConfigId = params(SimClustersANNParams.SimClustersANN4ConfigId) // SimClusters ANN Queries for different SANN clusters val simClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANNConfigId, params ) val experimentalSimClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, experimentalSimClustersANNConfigId, params ) val simClustersANN1Query = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANN1ConfigId, params ) val simClustersANN2Query = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANN2ConfigId, params ) val simClustersANN3Query = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANN3ConfigId, params ) val simClustersANN5Query = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANN5ConfigId, params ) val simClustersANN4Query = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANN4ConfigId, params ) // TweetBasedCandidateGeneration val maxCandidateNumPerSourceKey = params(GlobalParams.MaxCandidateNumPerSourceKeyParam) // TwHIN val twhinModelId = params(TweetBasedTwHINParams.ModelIdParam) val enableTwHIN = params(RelatedTweetTweetBasedParams.EnableTwHINParam) val twhinMaxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam) // QIG val enableQig = params(RelatedTweetTweetBasedParams.EnableQigSimilarTweetsParam) val qigMaxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam) val qigMaxNumSimilarTweets = params( TweetBasedCandidateGenerationParams.QigMaxNumSimilarTweetsParam) // UTG val enableUtg = params(RelatedTweetTweetBasedParams.EnableUTGParam) // UVG val enableUvg = params(RelatedTweetTweetBasedParams.EnableUVGParam) // SourceType.RequestTweetId is a placeholder. val sourceInfo = SourceInfo(SourceType.RequestTweetId, internalId, None) EngineQuery( Query( sourceInfo = sourceInfo, maxCandidateNumPerSourceKey = maxCandidateNumPerSourceKey, enableSimClustersANN = enableSimClustersANN, simClustersMinScore = simClustersMinScore, simClustersVideoBasedMinScore = simClustersMinScore, simClustersANNQuery = simClustersANNQuery, enableExperimentalSimClustersANN = enableExperimentalSimClustersANN, experimentalSimClustersANNQuery = experimentalSimClustersANNQuery, enableSimClustersANN1 = enableSimClustersANN1, simClustersANN1Query = simClustersANN1Query, enableSimClustersANN2 = enableSimClustersANN2, simClustersANN2Query = simClustersANN2Query, enableSimClustersANN3 = enableSimClustersANN3, simClustersANN3Query = simClustersANN3Query, enableSimClustersANN5 = enableSimClustersANN5, simClustersANN5Query = simClustersANN5Query, enableSimClustersANN4 = enableSimClustersANN4, simClustersANN4Query = simClustersANN4Query, twhinModelId = twhinModelId, enableTwHIN = enableTwHIN, twhinMaxTweetAgeHours = twhinMaxTweetAgeHours, qigMaxTweetAgeHours = qigMaxTweetAgeHours, qigMaxNumSimilarTweets = qigMaxNumSimilarTweets, enableUtg = enableUtg, utgQuery = TweetBasedUserTweetGraphSimilarityEngine .fromParams(sourceInfo.internalId, params), enableQig = enableQig, qigQuery = TweetBasedQigSimilarityEngine.fromParams(sourceInfo.internalId, params), enableUvg = enableUvg, uvgQuery = TweetBasedUserVideoGraphSimilarityEngine.fromParams(sourceInfo.internalId, params), params = params, ), params ) } def fromParamsForRelatedVideoTweet( internalId: InternalId, params: configapi.Params, ): EngineQuery[Query] = { // SimClusters val enableSimClustersANN = params(RelatedVideoTweetTweetBasedParams.EnableSimClustersANNParam) val simClustersModelVersion = ModelVersions.Enum.enumToSimClustersModelVersionMap(params(GlobalParams.ModelVersionParam)) val simClustersMinScore = params(RelatedVideoTweetTweetBasedParams.SimClustersMinScoreParam) val simClustersANNConfigId = params(SimClustersANNParams.SimClustersANNConfigId) val enableExperimentalSimClustersANN = params( RelatedVideoTweetTweetBasedParams.EnableExperimentalSimClustersANNParam) val experimentalSimClustersANNConfigId = params( SimClustersANNParams.ExperimentalSimClustersANNConfigId) // SimClusters - SANN cluster 1 Similarity Engine val enableSimClustersANN1 = params(RelatedVideoTweetTweetBasedParams.EnableSimClustersANN1Param) val simClustersANN1ConfigId = params(SimClustersANNParams.SimClustersANN1ConfigId) // SimClusters - SANN cluster 2 Similarity Engine val enableSimClustersANN2 = params(RelatedVideoTweetTweetBasedParams.EnableSimClustersANN2Param) val simClustersANN2ConfigId = params(SimClustersANNParams.SimClustersANN2ConfigId) // SimClusters - SANN cluster 3 Similarity Engine val enableSimClustersANN3 = params(RelatedVideoTweetTweetBasedParams.EnableSimClustersANN3Param) val simClustersANN3ConfigId = params(SimClustersANNParams.SimClustersANN3ConfigId) // SimClusters - SANN cluster 5 Similarity Engine val enableSimClustersANN5 = params(RelatedVideoTweetTweetBasedParams.EnableSimClustersANN5Param) val simClustersANN5ConfigId = params(SimClustersANNParams.SimClustersANN5ConfigId) // SimClusters - SANN cluster 4 Similarity Engine val enableSimClustersANN4 = params(RelatedVideoTweetTweetBasedParams.EnableSimClustersANN4Param) val simClustersANN4ConfigId = params(SimClustersANNParams.SimClustersANN4ConfigId) // SimClusters ANN Queries for different SANN clusters val simClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANNConfigId, params ) val experimentalSimClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, experimentalSimClustersANNConfigId, params ) val simClustersANN1Query = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANN1ConfigId, params ) val simClustersANN2Query = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANN2ConfigId, params ) val simClustersANN3Query = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANN3ConfigId, params ) val simClustersANN5Query = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANN5ConfigId, params ) val simClustersANN4Query = SimClustersANNSimilarityEngine.fromParams( internalId, EmbeddingType.LogFavLongestL2EmbeddingTweet, simClustersModelVersion, simClustersANN4ConfigId, params ) // TweetBasedCandidateGeneration val maxCandidateNumPerSourceKey = params(GlobalParams.MaxCandidateNumPerSourceKeyParam) // TwHIN val twhinModelId = params(TweetBasedTwHINParams.ModelIdParam) val enableTwHIN = params(RelatedVideoTweetTweetBasedParams.EnableTwHINParam) val twhinMaxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam) // QIG val enableQig = params(RelatedVideoTweetTweetBasedParams.EnableQigSimilarTweetsParam) val qigMaxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam) val qigMaxNumSimilarTweets = params( TweetBasedCandidateGenerationParams.QigMaxNumSimilarTweetsParam) // UTG val enableUtg = params(RelatedVideoTweetTweetBasedParams.EnableUTGParam) // SourceType.RequestTweetId is a placeholder. val sourceInfo = SourceInfo(SourceType.RequestTweetId, internalId, None) val enableUvg = params(RelatedVideoTweetTweetBasedParams.EnableUVGParam) EngineQuery( Query( sourceInfo = sourceInfo, maxCandidateNumPerSourceKey = maxCandidateNumPerSourceKey, enableSimClustersANN = enableSimClustersANN, simClustersMinScore = simClustersMinScore, simClustersVideoBasedMinScore = simClustersMinScore, simClustersANNQuery = simClustersANNQuery, enableExperimentalSimClustersANN = enableExperimentalSimClustersANN, experimentalSimClustersANNQuery = experimentalSimClustersANNQuery, enableSimClustersANN1 = enableSimClustersANN1, simClustersANN1Query = simClustersANN1Query, enableSimClustersANN2 = enableSimClustersANN2, simClustersANN2Query = simClustersANN2Query, enableSimClustersANN3 = enableSimClustersANN3, simClustersANN3Query = simClustersANN3Query, enableSimClustersANN5 = enableSimClustersANN5, simClustersANN5Query = simClustersANN5Query, enableSimClustersANN4 = enableSimClustersANN4, simClustersANN4Query = simClustersANN4Query, twhinModelId = twhinModelId, enableTwHIN = enableTwHIN, twhinMaxTweetAgeHours = twhinMaxTweetAgeHours, qigMaxTweetAgeHours = qigMaxTweetAgeHours, qigMaxNumSimilarTweets = qigMaxNumSimilarTweets, enableUtg = enableUtg, utgQuery = TweetBasedUserTweetGraphSimilarityEngine .fromParams(sourceInfo.internalId, params), enableUvg = enableUvg, uvgQuery = TweetBasedUserVideoGraphSimilarityEngine.fromParams(sourceInfo.internalId, params), enableQig = enableQig, qigQuery = TweetBasedQigSimilarityEngine.fromParams(sourceInfo.internalId, params), params = params ), params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserAdGraphSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.param.TweetBasedUserAdGraphParams import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.recos.user_ad_graph.thriftscala.ConsumersBasedRelatedAdRequest import com.twitter.recos.user_ad_graph.thriftscala.RelatedAdResponse import com.twitter.recos.user_ad_graph.thriftscala.UserAdGraph import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.twistly.thriftscala.TweetRecentEngagedUsers import com.twitter.util.Future import javax.inject.Singleton /** * This store looks for similar tweets from UserAdGraph for a Source TweetId * For a query tweet,User Ad Graph (UAG) * lets us find out which other tweets share a lot of the same engagers with the query tweet */ @Singleton case class TweetBasedUserAdGraphSimilarityEngine( userAdGraphService: UserAdGraph.MethodPerEndpoint, tweetEngagedUsersStore: ReadableStore[TweetId, TweetRecentEngagedUsers], statsReceiver: StatsReceiver) extends ReadableStore[ TweetBasedUserAdGraphSimilarityEngine.Query, Seq[TweetWithScore] ] { import TweetBasedUserAdGraphSimilarityEngine._ private val stats = statsReceiver.scope(this.getClass.getSimpleName) private val fetchCoverageExpansionCandidatesStat = stats.scope("fetchCoverageExpansionCandidates") override def get( query: TweetBasedUserAdGraphSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { query.sourceId match { case InternalId.TweetId(tweetId) => getCandidates(tweetId, query) case _ => Future.value(None) } } // We first fetch tweet's recent engaged users as consumeSeedSet from MH store, // then query consumersBasedUTG using the consumerSeedSet private def getCandidates( tweetId: TweetId, query: TweetBasedUserAdGraphSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { StatsUtil .trackOptionItemsStats(fetchCoverageExpansionCandidatesStat) { tweetEngagedUsersStore .get(tweetId).flatMap { _.map { tweetRecentEngagedUsers => val consumerSeedSet = tweetRecentEngagedUsers.recentEngagedUsers .map { _.userId }.take(query.maxConsumerSeedsNum) val consumersBasedRelatedAdRequest = ConsumersBasedRelatedAdRequest( consumerSeedSet = consumerSeedSet, maxResults = Some(query.maxResults), minCooccurrence = Some(query.minCooccurrence), excludeTweetIds = Some(Seq(tweetId)), minScore = Some(query.consumersBasedMinScore), maxTweetAgeInHours = Some(query.maxTweetAgeInHours) ) toTweetWithScore(userAdGraphService .consumersBasedRelatedAds(consumersBasedRelatedAdRequest).map { Some(_) }) }.getOrElse(Future.value(None)) } } } } object TweetBasedUserAdGraphSimilarityEngine { def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo = { SimilarityEngineInfo( similarityEngineType = SimilarityEngineType.TweetBasedUserAdGraph, modelId = None, score = Some(score)) } private def toTweetWithScore( relatedAdResponseFut: Future[Option[RelatedAdResponse]] ): Future[Option[Seq[TweetWithScore]]] = { relatedAdResponseFut.map { relatedAdResponseOpt => relatedAdResponseOpt.map { relatedAdResponse => val candidates = relatedAdResponse.adTweets.map(tweet => TweetWithScore(tweet.adTweetId, tweet.score)) candidates } } } case class Query( sourceId: InternalId, maxResults: Int, minCooccurrence: Int, consumersBasedMinScore: Double, maxTweetAgeInHours: Int, maxConsumerSeedsNum: Int, ) def fromParams( sourceId: InternalId, params: configapi.Params, ): EngineQuery[Query] = { EngineQuery( Query( sourceId = sourceId, maxResults = params(GlobalParams.MaxCandidateNumPerSourceKeyParam), minCooccurrence = params(TweetBasedUserAdGraphParams.MinCoOccurrenceParam), consumersBasedMinScore = params(TweetBasedUserAdGraphParams.ConsumersBasedMinScoreParam), maxTweetAgeInHours = params(GlobalParams.MaxTweetAgeHoursParam).inHours, maxConsumerSeedsNum = params(TweetBasedUserAdGraphParams.MaxConsumerSeedsNumParam), ), params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserTweetGraphSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.param.TweetBasedUserTweetGraphParams import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.recos.user_tweet_graph.thriftscala.RelatedTweetResponse import com.twitter.recos.user_tweet_graph.thriftscala.TweetBasedRelatedTweetRequest import com.twitter.recos.user_tweet_graph.thriftscala.ConsumersBasedRelatedTweetRequest import com.twitter.recos.user_tweet_graph.thriftscala.UserTweetGraph import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.twistly.thriftscala.TweetRecentEngagedUsers import com.twitter.util.Future import javax.inject.Singleton import com.twitter.snowflake.id.SnowflakeId import com.twitter.timelines.configapi import com.twitter.util.Duration import com.twitter.util.Time import scala.concurrent.duration.HOURS /** * This store looks for similar tweets from UserTweetGraph for a Source TweetId * For a query tweet,User Tweet Graph (UTG), * lets us find out which other tweets share a lot of the same engagers with the query tweet * one-pager: go/UTG */ @Singleton case class TweetBasedUserTweetGraphSimilarityEngine( userTweetGraphService: UserTweetGraph.MethodPerEndpoint, tweetEngagedUsersStore: ReadableStore[TweetId, TweetRecentEngagedUsers], statsReceiver: StatsReceiver) extends ReadableStore[ TweetBasedUserTweetGraphSimilarityEngine.Query, Seq[TweetWithScore] ] { import TweetBasedUserTweetGraphSimilarityEngine._ private val stats = statsReceiver.scope(this.getClass.getSimpleName) private val fetchCandidatesStat = stats.scope("fetchCandidates") private val fetchCoverageExpansionCandidatesStat = stats.scope("fetchCoverageExpansionCandidates") override def get( query: TweetBasedUserTweetGraphSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { query.sourceId match { case InternalId.TweetId(tweetId) if query.enableCoverageExpansionAllTweet => getCoverageExpansionCandidates(tweetId, query) case InternalId.TweetId(tweetId) if query.enableCoverageExpansionOldTweet => // For Home if (isOldTweet(tweetId)) getCoverageExpansionCandidates(tweetId, query) else getCandidates(tweetId, query) case InternalId.TweetId(tweetId) => getCandidates(tweetId, query) case _ => Future.value(None) } } // This is the main candidate source private def getCandidates( tweetId: TweetId, query: TweetBasedUserTweetGraphSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { StatsUtil.trackOptionItemsStats(fetchCandidatesStat) { val tweetBasedRelatedTweetRequest = { TweetBasedRelatedTweetRequest( tweetId, maxResults = Some(query.maxResults), minCooccurrence = Some(query.minCooccurrence), excludeTweetIds = Some(Seq(tweetId)), minScore = Some(query.tweetBasedMinScore), maxTweetAgeInHours = Some(query.maxTweetAgeInHours) ) } toTweetWithScore( userTweetGraphService.tweetBasedRelatedTweets(tweetBasedRelatedTweetRequest).map { Some(_) }) } } // function for DDGs, for coverage expansion algo, we first fetch tweet's recent engaged users as consumeSeedSet from MH store, // and query consumersBasedUTG using the consumeSeedSet private def getCoverageExpansionCandidates( tweetId: TweetId, query: TweetBasedUserTweetGraphSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { StatsUtil .trackOptionItemsStats(fetchCoverageExpansionCandidatesStat) { tweetEngagedUsersStore .get(tweetId).flatMap { _.map { tweetRecentEngagedUsers => val consumerSeedSet = tweetRecentEngagedUsers.recentEngagedUsers .map { _.userId }.take(query.maxConsumerSeedsNum) val consumersBasedRelatedTweetRequest = ConsumersBasedRelatedTweetRequest( consumerSeedSet = consumerSeedSet, maxResults = Some(query.maxResults), minCooccurrence = Some(query.minCooccurrence), excludeTweetIds = Some(Seq(tweetId)), minScore = Some(query.consumersBasedMinScore), maxTweetAgeInHours = Some(query.maxTweetAgeInHours) ) toTweetWithScore(userTweetGraphService .consumersBasedRelatedTweets(consumersBasedRelatedTweetRequest).map { Some(_) }) }.getOrElse(Future.value(None)) } } } } object TweetBasedUserTweetGraphSimilarityEngine { def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo = { SimilarityEngineInfo( similarityEngineType = SimilarityEngineType.TweetBasedUserTweetGraph, modelId = None, score = Some(score)) } private val oldTweetCap: Duration = Duration(48, HOURS) private def toTweetWithScore( relatedTweetResponseFut: Future[Option[RelatedTweetResponse]] ): Future[Option[Seq[TweetWithScore]]] = { relatedTweetResponseFut.map { relatedTweetResponseOpt => relatedTweetResponseOpt.map { relatedTweetResponse => val candidates = relatedTweetResponse.tweets.map(tweet => TweetWithScore(tweet.tweetId, tweet.score)) candidates } } } private def isOldTweet(tweetId: TweetId): Boolean = { SnowflakeId .timeFromIdOpt(tweetId).forall { tweetTime => tweetTime < Time.now - oldTweetCap } // If there's no snowflake timestamp, we have no idea when this tweet happened. } case class Query( sourceId: InternalId, maxResults: Int, minCooccurrence: Int, tweetBasedMinScore: Double, consumersBasedMinScore: Double, maxTweetAgeInHours: Int, maxConsumerSeedsNum: Int, enableCoverageExpansionOldTweet: Boolean, enableCoverageExpansionAllTweet: Boolean, ) def fromParams( sourceId: InternalId, params: configapi.Params, ): EngineQuery[Query] = { EngineQuery( Query( sourceId = sourceId, maxResults = params(GlobalParams.MaxCandidateNumPerSourceKeyParam), minCooccurrence = params(TweetBasedUserTweetGraphParams.MinCoOccurrenceParam), tweetBasedMinScore = params(TweetBasedUserTweetGraphParams.TweetBasedMinScoreParam), consumersBasedMinScore = params(TweetBasedUserTweetGraphParams.ConsumersBasedMinScoreParam), maxTweetAgeInHours = params(GlobalParams.MaxTweetAgeHoursParam).inHours, maxConsumerSeedsNum = params(TweetBasedUserTweetGraphParams.MaxConsumerSeedsNumParam), enableCoverageExpansionOldTweet = params(TweetBasedUserTweetGraphParams.EnableCoverageExpansionOldTweetParam), enableCoverageExpansionAllTweet = params(TweetBasedUserTweetGraphParams.EnableCoverageExpansionAllTweetParam), ), params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserVideoGraphSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.param.TweetBasedUserVideoGraphParams import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.util.StatsUtil import com.twitter.recos.user_video_graph.thriftscala.RelatedTweetResponse import com.twitter.recos.user_video_graph.thriftscala.ConsumersBasedRelatedTweetRequest import com.twitter.recos.user_video_graph.thriftscala.TweetBasedRelatedTweetRequest import com.twitter.recos.user_video_graph.thriftscala.UserVideoGraph import com.twitter.simclusters_v2.common.TweetId import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.snowflake.id.SnowflakeId import com.twitter.timelines.configapi import com.twitter.twistly.thriftscala.TweetRecentEngagedUsers import com.twitter.util.Duration import javax.inject.Singleton import com.twitter.util.Future import com.twitter.util.Time import scala.concurrent.duration.HOURS /** * This store looks for similar tweets from UserVideoGraph for a Source TweetId * For a query tweet,User Video Graph (UVG), * lets us find out which other video tweets share a lot of the same engagers with the query tweet */ @Singleton case class TweetBasedUserVideoGraphSimilarityEngine( userVideoGraphService: UserVideoGraph.MethodPerEndpoint, tweetEngagedUsersStore: ReadableStore[TweetId, TweetRecentEngagedUsers], statsReceiver: StatsReceiver) extends ReadableStore[ TweetBasedUserVideoGraphSimilarityEngine.Query, Seq[TweetWithScore] ] { import TweetBasedUserVideoGraphSimilarityEngine._ private val stats = statsReceiver.scope(this.getClass.getSimpleName) private val fetchCandidatesStat = stats.scope("fetchCandidates") private val fetchCoverageExpansionCandidatesStat = stats.scope("fetchCoverageExpansionCandidates") override def get( query: TweetBasedUserVideoGraphSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { query.sourceId match { case InternalId.TweetId(tweetId) if query.enableCoverageExpansionAllTweet => getCoverageExpansionCandidates(tweetId, query) case InternalId.TweetId(tweetId) if query.enableCoverageExpansionOldTweet => // For Home if (isOldTweet(tweetId)) getCoverageExpansionCandidates(tweetId, query) else getCandidates(tweetId, query) case InternalId.TweetId(tweetId) => getCandidates(tweetId, query) case _ => Future.value(None) } } private def getCandidates( tweetId: TweetId, query: TweetBasedUserVideoGraphSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { StatsUtil.trackOptionItemsStats(fetchCandidatesStat) { val tweetBasedRelatedTweetRequest = { TweetBasedRelatedTweetRequest( tweetId, maxResults = Some(query.maxResults), minCooccurrence = Some(query.minCooccurrence), excludeTweetIds = Some(Seq(tweetId)), minScore = Some(query.tweetBasedMinScore), maxTweetAgeInHours = Some(query.maxTweetAgeInHours) ) } toTweetWithScore( userVideoGraphService.tweetBasedRelatedTweets(tweetBasedRelatedTweetRequest).map { Some(_) }) } } private def getCoverageExpansionCandidates( tweetId: TweetId, query: TweetBasedUserVideoGraphSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { StatsUtil .trackOptionItemsStats(fetchCoverageExpansionCandidatesStat) { tweetEngagedUsersStore .get(tweetId).flatMap { _.map { tweetRecentEngagedUsers => val consumerSeedSet = tweetRecentEngagedUsers.recentEngagedUsers .map { _.userId }.take(query.maxConsumerSeedsNum) val consumersBasedRelatedTweetRequest = ConsumersBasedRelatedTweetRequest( consumerSeedSet = consumerSeedSet, maxResults = Some(query.maxResults), minCooccurrence = Some(query.minCooccurrence), excludeTweetIds = Some(Seq(tweetId)), minScore = Some(query.consumersBasedMinScore), maxTweetAgeInHours = Some(query.maxTweetAgeInHours) ) toTweetWithScore(userVideoGraphService .consumersBasedRelatedTweets(consumersBasedRelatedTweetRequest).map { Some(_) }) }.getOrElse(Future.value(None)) } } } } object TweetBasedUserVideoGraphSimilarityEngine { private val oldTweetCap: Duration = Duration(24, HOURS) def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo = { SimilarityEngineInfo( similarityEngineType = SimilarityEngineType.TweetBasedUserVideoGraph, modelId = None, score = Some(score)) } private def toTweetWithScore( relatedTweetResponseFut: Future[Option[RelatedTweetResponse]] ): Future[Option[Seq[TweetWithScore]]] = { relatedTweetResponseFut.map { relatedTweetResponseOpt => relatedTweetResponseOpt.map { relatedTweetResponse => val candidates = relatedTweetResponse.tweets.map(tweet => TweetWithScore(tweet.tweetId, tweet.score)) candidates } } } private def isOldTweet(tweetId: TweetId): Boolean = { SnowflakeId .timeFromIdOpt(tweetId).forall { tweetTime => tweetTime < Time.now - oldTweetCap } // If there's no snowflake timestamp, we have no idea when this tweet happened. } case class Query( sourceId: InternalId, maxResults: Int, minCooccurrence: Int, tweetBasedMinScore: Double, consumersBasedMinScore: Double, maxTweetAgeInHours: Int, maxConsumerSeedsNum: Int, enableCoverageExpansionOldTweet: Boolean, enableCoverageExpansionAllTweet: Boolean) def fromParams( sourceId: InternalId, params: configapi.Params, ): EngineQuery[Query] = { EngineQuery( Query( sourceId = sourceId, maxResults = params(GlobalParams.MaxCandidateNumPerSourceKeyParam), minCooccurrence = params(TweetBasedUserVideoGraphParams.MinCoOccurrenceParam), tweetBasedMinScore = params(TweetBasedUserVideoGraphParams.TweetBasedMinScoreParam), consumersBasedMinScore = params(TweetBasedUserVideoGraphParams.ConsumersBasedMinScoreParam), maxTweetAgeInHours = params(GlobalParams.MaxTweetAgeHoursParam).inHours, maxConsumerSeedsNum = params(TweetBasedUserVideoGraphParams.MaxConsumerSeedsNumParam), enableCoverageExpansionOldTweet = params(TweetBasedUserVideoGraphParams.EnableCoverageExpansionOldTweetParam), enableCoverageExpansionAllTweet = params(TweetBasedUserVideoGraphParams.EnableCoverageExpansionAllTweetParam) ), params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TwhinCollabFilterSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.simclusters_v2.common.TweetId import com.twitter.cr_mixer.model.TweetWithScore import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.util.Future import javax.inject.Singleton @Singleton case class TwhinCollabFilterSimilarityEngine( twhinCandidatesStratoStore: ReadableStore[Long, Seq[TweetId]], statsReceiver: StatsReceiver) extends ReadableStore[ TwhinCollabFilterSimilarityEngine.Query, Seq[TweetWithScore] ] { import TwhinCollabFilterSimilarityEngine._ override def get( query: TwhinCollabFilterSimilarityEngine.Query ): Future[Option[Seq[TweetWithScore]]] = { query.sourceId match { case InternalId.UserId(userId) => twhinCandidatesStratoStore.get(userId).map { _.map { _.map { tweetId => TweetWithScore(tweetId, defaultScore) } } } case _ => Future.None } } } object TwhinCollabFilterSimilarityEngine { val defaultScore: Double = 1.0 case class TwhinCollabFilterView(clusterVersion: String) case class Query( sourceId: InternalId, ) def toSimilarityEngineInfo( query: LookupEngineQuery[Query], score: Double ): SimilarityEngineInfo = { SimilarityEngineInfo( similarityEngineType = SimilarityEngineType.TwhinCollabFilter, modelId = Some(query.lookupKey), score = Some(score)) } def fromParams( sourceId: InternalId, modelId: String, params: configapi.Params, ): LookupEngineQuery[Query] = { LookupEngineQuery( Query(sourceId = sourceId), modelId, params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/UserTweetEntityGraphSimilarityEngine.scala ================================================ package com.twitter.cr_mixer.similarity_engine import com.twitter.recos.recos_common.thriftscala.SocialProofType import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.TweetWithScoreAndSocialProof import com.twitter.cr_mixer.param.UtegTweetGlobalParams import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.finagle.stats.StatsReceiver import com.twitter.recos.user_tweet_entity_graph.thriftscala.TweetEntityDisplayLocation import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph import com.twitter.recos.user_tweet_entity_graph.thriftscala.RecommendTweetEntityRequest import com.twitter.recos.user_tweet_entity_graph.thriftscala.RecommendationType import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityRecommendationUnion.TweetRec import com.twitter.simclusters_v2.common.UserId import com.twitter.simclusters_v2.common.TweetId import com.twitter.storehaus.ReadableStore import com.twitter.timelines.configapi import com.twitter.util.Duration import com.twitter.util.Future import javax.inject.Singleton @Singleton case class UserTweetEntityGraphSimilarityEngine( userTweetEntityGraph: UserTweetEntityGraph.MethodPerEndpoint, statsReceiver: StatsReceiver) extends ReadableStore[ UserTweetEntityGraphSimilarityEngine.Query, Seq[TweetWithScoreAndSocialProof] ] { override def get( query: UserTweetEntityGraphSimilarityEngine.Query ): Future[Option[Seq[TweetWithScoreAndSocialProof]]] = { val recommendTweetEntityRequest = RecommendTweetEntityRequest( requesterId = query.userId, displayLocation = TweetEntityDisplayLocation.HomeTimeline, recommendationTypes = Seq(RecommendationType.Tweet), seedsWithWeights = query.seedsWithWeights, maxResultsByType = Some(Map(RecommendationType.Tweet -> query.maxUtegCandidates)), maxTweetAgeInMillis = Some(query.maxTweetAge.inMilliseconds), excludedTweetIds = query.excludedTweetIds, maxUserSocialProofSize = Some(UserTweetEntityGraphSimilarityEngine.MaxUserSocialProofSize), maxTweetSocialProofSize = Some(UserTweetEntityGraphSimilarityEngine.MaxTweetSocialProofSize), minUserSocialProofSizes = Some(Map(RecommendationType.Tweet -> 1)), tweetTypes = None, socialProofTypes = query.socialProofTypes, socialProofTypeUnions = None, tweetAuthors = None, maxEngagementAgeInMillis = None, excludedTweetAuthors = None, ) userTweetEntityGraph .recommendTweets(recommendTweetEntityRequest) .map { recommendTweetsResponse => val candidates = recommendTweetsResponse.recommendations.flatMap { case TweetRec(recommendation) => Some( TweetWithScoreAndSocialProof( recommendation.tweetId, recommendation.score, recommendation.socialProofByType.toMap)) case _ => None } Some(candidates) } } } object UserTweetEntityGraphSimilarityEngine { private val MaxUserSocialProofSize = 10 private val MaxTweetSocialProofSize = 10 def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo = { SimilarityEngineInfo( similarityEngineType = SimilarityEngineType.Uteg, modelId = None, score = Some(score)) } case class Query( userId: UserId, seedsWithWeights: Map[UserId, Double], excludedTweetIds: Option[Seq[Long]] = None, maxUtegCandidates: Int, maxTweetAge: Duration, socialProofTypes: Option[Seq[SocialProofType]]) def fromParams( userId: UserId, seedsWithWeights: Map[UserId, Double], excludedTweetIds: Option[Seq[TweetId]] = None, params: configapi.Params, ): EngineQuery[Query] = { EngineQuery( Query( userId = userId, seedsWithWeights = seedsWithWeights, excludedTweetIds = excludedTweetIds, maxUtegCandidates = params(UtegTweetGlobalParams.MaxUtegCandidatesToRequestParam), maxTweetAge = params(UtegTweetGlobalParams.CandidateRefreshSinceTimeOffsetHoursParam), socialProofTypes = Some(Seq(SocialProofType.Favorite)) ), params ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/storehaus:core", "3rdparty/jvm/javax/inject:javax.inject", "3rdparty/src/jvm/com/twitter/storehaus:core", "configapi/configapi-core", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/config", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider", "cr-mixer/thrift/src/main/thrift:thrift-scala", "decider/src/main/scala", "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "frigate/frigate-common:base", "frigate/frigate-common:util", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/base", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/candidate", "src/scala/com/twitter/simclusters_v2/common", "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", "src/thrift/com/twitter/hermit/stp:hermit-stp-scala", "src/thrift/com/twitter/onboarding/relevance/coldstart_lookalike:coldstartlookalike-thrift-scala", "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", "user-signal-service/thrift/src/main/thrift:thrift-scala", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/FrsSourceGraphFetcher.scala ================================================ package com.twitter.cr_mixer.source_signal import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.GraphSourceInfo import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.param.FrsParams import com.twitter.cr_mixer.source_signal.FrsStore.FrsQueryResult import com.twitter.cr_mixer.source_signal.SourceFetcher.FetcherQuery import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.finagle.stats.StatsReceiver import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton /*** * This store fetches user recommendations from FRS (go/frs) for a given userId */ @Singleton case class FrsSourceGraphFetcher @Inject() ( @Named(ModuleNames.FrsStore) frsStore: ReadableStore[FrsStore.Query, Seq[FrsQueryResult]], override val timeoutConfig: TimeoutConfig, globalStats: StatsReceiver) extends SourceGraphFetcher { override protected val stats: StatsReceiver = globalStats.scope(identifier) override protected val graphSourceType: SourceType = SourceType.FollowRecommendation override def isEnabled(query: FetcherQuery): Boolean = { query.params(FrsParams.EnableSourceGraphParam) } override def fetchAndProcess( query: FetcherQuery, ): Future[Option[GraphSourceInfo]] = { val rawSignals = trackPerItemStats(query)( frsStore .get( FrsStore .Query(query.userId, query.params(FrsParams.MaxConsumerSeedsNumParam))).map { _.map { _.map { v => (v.userId, v.score) } } } ) rawSignals.map { _.map { userWithScores => convertGraphSourceInfo(userWithScores) } } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/FrsSourceSignalFetcher.scala ================================================ package com.twitter.cr_mixer.source_signal import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.param.FrsParams import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.source_signal.FrsStore.FrsQueryResult import com.twitter.cr_mixer.source_signal.SourceFetcher.FetcherQuery import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.finagle.stats.StatsReceiver import com.twitter.simclusters_v2.common.UserId import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import javax.inject.Singleton import javax.inject.Inject import javax.inject.Named @Singleton case class FrsSourceSignalFetcher @Inject() ( @Named(ModuleNames.FrsStore) frsStore: ReadableStore[FrsStore.Query, Seq[FrsQueryResult]], override val timeoutConfig: TimeoutConfig, globalStats: StatsReceiver) extends SourceSignalFetcher { override protected val stats: StatsReceiver = globalStats.scope(identifier) override type SignalConvertType = UserId override def isEnabled(query: FetcherQuery): Boolean = { query.params(FrsParams.EnableSourceParam) } override def fetchAndProcess(query: FetcherQuery): Future[Option[Seq[SourceInfo]]] = { // Fetch raw signals val rawSignals = frsStore .get(FrsStore.Query(query.userId, query.params(GlobalParams.UnifiedMaxSourceKeyNum))) .map { _.map { _.map { _.userId } } } // Process signals rawSignals.map { _.map { frsUsers => convertSourceInfo(SourceType.FollowRecommendation, frsUsers) } } } override def convertSourceInfo( sourceType: SourceType, signals: Seq[SignalConvertType] ): Seq[SourceInfo] = { signals.map { signal => SourceInfo( sourceType = sourceType, internalId = InternalId.UserId(signal), sourceEventTime = None ) } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/FrsStore.scala ================================================ package com.twitter.cr_mixer.source_signal import com.twitter.cr_mixer.param.decider.CrMixerDecider import com.twitter.cr_mixer.param.decider.DeciderConstants import com.twitter.cr_mixer.source_signal.FrsStore.Query import com.twitter.cr_mixer.source_signal.FrsStore.FrsQueryResult import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.thriftscala.ClientContext import com.twitter.follow_recommendations.thriftscala.DisplayLocation import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService import com.twitter.follow_recommendations.thriftscala.Recommendation import com.twitter.follow_recommendations.thriftscala.RecommendationRequest import com.twitter.storehaus.ReadableStore import javax.inject.Singleton import com.twitter.simclusters_v2.common.UserId import com.twitter.util.Future @Singleton case class FrsStore( frsClient: FollowRecommendationsThriftService.MethodPerEndpoint, statsReceiver: StatsReceiver, decider: CrMixerDecider) extends ReadableStore[Query, Seq[FrsQueryResult]] { override def get( query: Query ): Future[Option[Seq[FrsQueryResult]]] = { if (decider.isAvailable(DeciderConstants.enableFRSTrafficDeciderKey)) { val recommendationRequest = buildFollowRecommendationRequest(query) frsClient .getRecommendations(recommendationRequest).map { recommendationResponse => Some(recommendationResponse.recommendations.collect { case recommendation: Recommendation.User => FrsQueryResult( recommendation.user.userId, recommendation.user.scoringDetails .flatMap(_.score).getOrElse(0.0), recommendation.user.scoringDetails .flatMap(_.candidateSourceDetails.flatMap(_.primarySource)), recommendation.user.scoringDetails .flatMap(_.candidateSourceDetails.flatMap(_.candidateSourceScores)).map(_.toMap) ) }) } } else { Future.None } } private def buildFollowRecommendationRequest( query: Query ): RecommendationRequest = { RecommendationRequest( clientContext = ClientContext( userId = Some(query.userId), countryCode = query.countryCodeOpt, languageCode = query.languageCodeOpt), displayLocation = query.displayLocation, maxResults = Some(query.maxConsumerSeedsNum), excludedIds = Some(query.excludedUserIds) ) } } object FrsStore { case class Query( userId: UserId, maxConsumerSeedsNum: Int, displayLocation: DisplayLocation = DisplayLocation.ContentRecommender, excludedUserIds: Seq[UserId] = Seq.empty, languageCodeOpt: Option[String] = None, countryCodeOpt: Option[String] = None) case class FrsQueryResult( userId: UserId, score: Double, primarySource: Option[Int], sourceWithScores: Option[Map[String, Double]]) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/RealGraphInSourceGraphFetcher.scala ================================================ package com.twitter.cr_mixer.source_signal import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.GraphSourceInfo import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.param.RealGraphInParams import com.twitter.cr_mixer.source_signal.SourceFetcher.FetcherQuery import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.finagle.stats.StatsReceiver import com.twitter.simclusters_v2.common.UserId import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import com.twitter.wtf.candidate.thriftscala.CandidateSeq import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton /** * This store fetch user recommendations from In-Network RealGraph (go/realgraph) for a given userId */ @Singleton case class RealGraphInSourceGraphFetcher @Inject() ( @Named(ModuleNames.RealGraphInStore) realGraphStoreMh: ReadableStore[UserId, CandidateSeq], override val timeoutConfig: TimeoutConfig, globalStats: StatsReceiver) extends SourceGraphFetcher { override protected val stats: StatsReceiver = globalStats.scope(identifier) override protected val graphSourceType: SourceType = SourceType.RealGraphIn override def isEnabled(query: FetcherQuery): Boolean = { query.params(RealGraphInParams.EnableSourceGraphParam) } override def fetchAndProcess( query: FetcherQuery, ): Future[Option[GraphSourceInfo]] = { val rawSignals = trackPerItemStats(query)( realGraphStoreMh.get(query.userId).map { _.map { candidateSeq => candidateSeq.candidates .map { candidate => // Bundle the userId with its score (candidate.userId, candidate.score) } } } ) rawSignals.map { _.map { userWithScores => convertGraphSourceInfo(userWithScores) } } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/RealGraphOonSourceGraphFetcher.scala ================================================ package com.twitter.cr_mixer.source_signal import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.GraphSourceInfo import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.param.RealGraphOonParams import com.twitter.cr_mixer.source_signal.SourceFetcher.FetcherQuery import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.finagle.stats.StatsReceiver import com.twitter.simclusters_v2.common.UserId import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import com.twitter.wtf.candidate.thriftscala.CandidateSeq import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton /** * This store fetch user recommendations from RealGraphOON (go/realgraph) for a given userId */ @Singleton case class RealGraphOonSourceGraphFetcher @Inject() ( @Named(ModuleNames.RealGraphOonStore) realGraphOonStore: ReadableStore[UserId, CandidateSeq], override val timeoutConfig: TimeoutConfig, globalStats: StatsReceiver) extends SourceGraphFetcher { override protected val stats: StatsReceiver = globalStats.scope(identifier) override protected val graphSourceType: SourceType = SourceType.RealGraphOon override def isEnabled(query: FetcherQuery): Boolean = { query.params(RealGraphOonParams.EnableSourceGraphParam) } override def fetchAndProcess( query: FetcherQuery, ): Future[Option[GraphSourceInfo]] = { val rawSignals = trackPerItemStats(query)( realGraphOonStore.get(query.userId).map { _.map { candidateSeq => candidateSeq.candidates .map { candidate => // Bundle the userId with its score (candidate.userId, candidate.score) }.take(query.params(RealGraphOonParams.MaxConsumerSeedsNumParam)) } } ) rawSignals.map { _.map { userWithScores => convertGraphSourceInfo(userWithScores) } } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/SourceFetcher.scala ================================================ package com.twitter.cr_mixer.source_signal import com.twitter.core_workflows.user_model.thriftscala.UserState import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.source_signal.SourceFetcher.FetcherQuery import com.twitter.simclusters_v2.common.UserId import com.twitter.timelines.configapi.Params import com.twitter.cr_mixer.thriftscala.{Product => TProduct} import com.twitter.finagle.GlobalRequestTimeoutException import com.twitter.finagle.mux.ClientDiscardedRequestException import com.twitter.finagle.mux.ServerApplicationError import com.twitter.finagle.stats.StatsReceiver import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import com.twitter.util.TimeoutException import org.apache.thrift.TApplicationException import com.twitter.util.logging.Logging /** * A SourceFetcher is a trait which, given a [[FetcherQuery]], returns [[ResultType]] * The main purposes of a SourceFetcher is to provide a consistent interface for source fetch * logic, and provides default functions, including: * - Identification * - Observability * - Timeout settings * - Exception Handling */ trait SourceFetcher[ResultType] extends ReadableStore[FetcherQuery, ResultType] with Logging { protected final val timer = com.twitter.finagle.util.DefaultTimer protected final def identifier: String = this.getClass.getSimpleName protected def stats: StatsReceiver protected def timeoutConfig: TimeoutConfig /*** * Use FeatureSwitch to decide if a specific source is enabled. */ def isEnabled(query: FetcherQuery): Boolean /*** * This function fetches the raw sources and process them. * Custom stats tracking can be added depending on the type of ResultType */ def fetchAndProcess( query: FetcherQuery, ): Future[Option[ResultType]] /*** * Side-effect function to track stats for signal fetching and processing. */ def trackStats( query: FetcherQuery )( func: => Future[Option[ResultType]] ): Future[Option[ResultType]] /*** * This function is called by the top level class to fetch sources. It executes the pipeline to * fetch raw data, process and transform the sources. Exceptions, Stats, and timeout control are * handled here. */ override def get( query: FetcherQuery ): Future[Option[ResultType]] = { val scopedStats = stats.scope(query.product.originalName) if (isEnabled(query)) { scopedStats.counter("gate_enabled").incr() trackStats(query)(fetchAndProcess(query)) .raiseWithin(timeoutConfig.signalFetchTimeout)(timer) .onFailure { e => scopedStats.scope("exceptions").counter(e.getClass.getSimpleName).incr() } .rescue { case _: TimeoutException | _: GlobalRequestTimeoutException | _: TApplicationException | _: ClientDiscardedRequestException | _: ServerApplicationError // TApplicationException inside => Future.None case e => logger.info(e) Future.None } } else { scopedStats.counter("gate_disabled").incr() Future.None } } } object SourceFetcher { /*** * Every SourceFetcher all share the same input: FetcherQuery */ case class FetcherQuery( userId: UserId, product: TProduct, userState: UserState, params: Params) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/SourceGraphFetcher.scala ================================================ package com.twitter.cr_mixer.source_signal import com.twitter.cr_mixer.model.GraphSourceInfo import com.twitter.cr_mixer.source_signal.SourceFetcher.FetcherQuery import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.frigate.common.util.StatsUtil import com.twitter.simclusters_v2.common.UserId import com.twitter.util.Future /*** * A SourceGraphFetcher is a trait that extends from `SourceFetcher` * and is specialized in tackling User Graph (eg., RealGraphOon, FRS) fetch. * * The [[ResultType]] of a SourceGraphFetcher is a `GraphSourceInfo` which contains a userSeedSet. * When we pass in userId, the underlying store returns one GraphSourceInfo. */ trait SourceGraphFetcher extends SourceFetcher[GraphSourceInfo] { protected final val DefaultSeedScore = 1.0 protected def graphSourceType: SourceType /*** * RawDataType contains a consumers seed UserId and a score (weight) */ protected type RawDataType = (UserId, Double) def trackStats( query: FetcherQuery )( func: => Future[Option[GraphSourceInfo]] ): Future[Option[GraphSourceInfo]] = { val productScopedStats = stats.scope(query.product.originalName) val productUserStateScopedStats = productScopedStats.scope(query.userState.toString) StatsUtil .trackOptionStats(productScopedStats) { StatsUtil .trackOptionStats(productUserStateScopedStats) { func } } } // Track per item stats on the fetched graph results def trackPerItemStats( query: FetcherQuery )( func: => Future[Option[Seq[RawDataType]]] ): Future[Option[Seq[RawDataType]]] = { val productScopedStats = stats.scope(query.product.originalName) val productUserStateScopedStats = productScopedStats.scope(query.userState.toString) StatsUtil.trackOptionItemsStats(productScopedStats) { StatsUtil.trackOptionItemsStats(productUserStateScopedStats) { func } } } /*** * Convert Seq[RawDataType] into GraphSourceInfo */ protected final def convertGraphSourceInfo( userWithScores: Seq[RawDataType] ): GraphSourceInfo = { GraphSourceInfo( sourceType = graphSourceType, seedWithScores = userWithScores.map { userWithScore => userWithScore._1 -> userWithScore._2 }.toMap ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/SourceInfoRouter.scala ================================================ package com.twitter.cr_mixer.source_signal import com.twitter.core_workflows.user_model.thriftscala.UserState import com.twitter.cr_mixer.model.GraphSourceInfo import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.source_signal.SourceFetcher.FetcherQuery import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.cr_mixer.thriftscala.{Product => TProduct} import com.twitter.simclusters_v2.common.UserId import com.twitter.timelines.configapi import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton @Singleton case class SourceInfoRouter @Inject() ( ussSourceSignalFetcher: UssSourceSignalFetcher, frsSourceSignalFetcher: FrsSourceSignalFetcher, frsSourceGraphFetcher: FrsSourceGraphFetcher, realGraphOonSourceGraphFetcher: RealGraphOonSourceGraphFetcher, realGraphInSourceGraphFetcher: RealGraphInSourceGraphFetcher, ) { def get( userId: UserId, product: TProduct, userState: UserState, params: configapi.Params ): Future[(Set[SourceInfo], Map[String, Option[GraphSourceInfo]])] = { val fetcherQuery = FetcherQuery(userId, product, userState, params) Future.join( getSourceSignals(fetcherQuery), getSourceGraphs(fetcherQuery) ) } private def getSourceSignals( fetcherQuery: FetcherQuery ): Future[Set[SourceInfo]] = { Future .join( ussSourceSignalFetcher.get(fetcherQuery), frsSourceSignalFetcher.get(fetcherQuery)).map { case (ussSignalsOpt, frsSignalsOpt) => (ussSignalsOpt.getOrElse(Seq.empty) ++ frsSignalsOpt.getOrElse(Seq.empty)).toSet } } private def getSourceGraphs( fetcherQuery: FetcherQuery ): Future[Map[String, Option[GraphSourceInfo]]] = { Future .join( frsSourceGraphFetcher.get(fetcherQuery), realGraphOonSourceGraphFetcher.get(fetcherQuery), realGraphInSourceGraphFetcher.get(fetcherQuery) ).map { case (frsGraphOpt, realGraphOonGraphOpt, realGraphInGraphOpt) => Map( SourceType.FollowRecommendation.name -> frsGraphOpt, SourceType.RealGraphOon.name -> realGraphOonGraphOpt, SourceType.RealGraphIn.name -> realGraphInGraphOpt, ) } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/SourceSignalFetcher.scala ================================================ package com.twitter.cr_mixer.source_signal import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.source_signal.SourceFetcher.FetcherQuery import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.frigate.common.util.StatsUtil import com.twitter.util.Future /*** * A SourceSignalFetcher is a trait that extends from `SourceFetcher` * and is specialized in tackling Signals (eg., USS, FRS) fetch. * Currently, we define Signals as (but not limited to) a set of past engagements that * the user makes, such as RecentFav, RecentFollow, etc. * * The [[ResultType]] of a SourceSignalFetcher is `Seq[SourceInfo]`. When we pass in userId, * the underlying store returns a list of signals. */ trait SourceSignalFetcher extends SourceFetcher[Seq[SourceInfo]] { protected type SignalConvertType def trackStats( query: FetcherQuery )( func: => Future[Option[Seq[SourceInfo]]] ): Future[Option[Seq[SourceInfo]]] = { val productScopedStats = stats.scope(query.product.originalName) val productUserStateScopedStats = productScopedStats.scope(query.userState.toString) StatsUtil .trackOptionItemsStats(productScopedStats) { StatsUtil .trackOptionItemsStats(productUserStateScopedStats) { func } } } /*** * Convert a list of Signals of type [[SignalConvertType]] into SourceInfo */ def convertSourceInfo( sourceType: SourceType, signals: Seq[SignalConvertType] ): Seq[SourceInfo] } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/UssSourceSignalFetcher.scala ================================================ package com.twitter.cr_mixer.source_signal import com.twitter.cr_mixer.config.TimeoutConfig import com.twitter.cr_mixer.model.ModuleNames import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.cr_mixer.source_signal.SourceFetcher.FetcherQuery import com.twitter.finagle.stats.StatsReceiver import com.twitter.storehaus.ReadableStore import com.twitter.usersignalservice.thriftscala.{Signal => UssSignal} import com.twitter.usersignalservice.thriftscala.SignalType import com.twitter.frigate.common.util.StatsUtil.Size import com.twitter.frigate.common.util.StatsUtil.Success import com.twitter.frigate.common.util.StatsUtil.Empty import com.twitter.util.Future import com.twitter.util.Time import javax.inject.Singleton import javax.inject.Inject import javax.inject.Named @Singleton case class UssSourceSignalFetcher @Inject() ( @Named(ModuleNames.UssStore) ussStore: ReadableStore[UssStore.Query, Seq[ (SignalType, Seq[UssSignal]) ]], override val timeoutConfig: TimeoutConfig, globalStats: StatsReceiver) extends SourceSignalFetcher { override protected val stats: StatsReceiver = globalStats.scope(identifier) override type SignalConvertType = UssSignal // always enable USS call. We have fine-grained FS to decider which signal to fetch override def isEnabled(query: FetcherQuery): Boolean = true override def fetchAndProcess( query: FetcherQuery, ): Future[Option[Seq[SourceInfo]]] = { // Fetch raw signals val rawSignals = ussStore.get(UssStore.Query(query.userId, query.params, query.product)).map { _.map { _.map { case (signalType, signals) => trackUssSignalStatsPerSignalType(query, signalType, signals) (signalType, signals) } } } /** * Process signals: * Transform a Seq of USS Signals with signalType specified to a Seq of SourceInfo * We do case match to make sure the SignalType can correctly map to a SourceType defined in CrMixer * and it should be simplified. */ rawSignals.map { _.map { nestedSignal => val sourceInfoList = nestedSignal.flatMap { case (signalType, ussSignals) => signalType match { case SignalType.TweetFavorite => convertSourceInfo(sourceType = SourceType.TweetFavorite, signals = ussSignals) case SignalType.Retweet => convertSourceInfo(sourceType = SourceType.Retweet, signals = ussSignals) case SignalType.Reply => convertSourceInfo(sourceType = SourceType.Reply, signals = ussSignals) case SignalType.OriginalTweet => convertSourceInfo(sourceType = SourceType.OriginalTweet, signals = ussSignals) case SignalType.AccountFollow => convertSourceInfo(sourceType = SourceType.UserFollow, signals = ussSignals) case SignalType.RepeatedProfileVisit180dMinVisit6V1 | SignalType.RepeatedProfileVisit90dMinVisit6V1 | SignalType.RepeatedProfileVisit14dMinVisit2V1 => convertSourceInfo( sourceType = SourceType.UserRepeatedProfileVisit, signals = ussSignals) case SignalType.NotificationOpenAndClickV1 => convertSourceInfo(sourceType = SourceType.NotificationClick, signals = ussSignals) case SignalType.TweetShareV1 => convertSourceInfo(sourceType = SourceType.TweetShare, signals = ussSignals) case SignalType.RealGraphOon => convertSourceInfo(sourceType = SourceType.RealGraphOon, signals = ussSignals) case SignalType.GoodTweetClick | SignalType.GoodTweetClick5s | SignalType.GoodTweetClick10s | SignalType.GoodTweetClick30s => convertSourceInfo(sourceType = SourceType.GoodTweetClick, signals = ussSignals) case SignalType.VideoView90dPlayback50V1 => convertSourceInfo( sourceType = SourceType.VideoTweetPlayback50, signals = ussSignals) case SignalType.VideoView90dQualityV1 => convertSourceInfo( sourceType = SourceType.VideoTweetQualityView, signals = ussSignals) case SignalType.GoodProfileClick | SignalType.GoodProfileClick20s | SignalType.GoodProfileClick30s => convertSourceInfo(sourceType = SourceType.GoodProfileClick, signals = ussSignals) // negative signals case SignalType.AccountBlock => convertSourceInfo(sourceType = SourceType.AccountBlock, signals = ussSignals) case SignalType.AccountMute => convertSourceInfo(sourceType = SourceType.AccountMute, signals = ussSignals) case SignalType.TweetReport => convertSourceInfo(sourceType = SourceType.TweetReport, signals = ussSignals) case SignalType.TweetDontLike => convertSourceInfo(sourceType = SourceType.TweetDontLike, signals = ussSignals) // Aggregated Signals case SignalType.TweetBasedUnifiedEngagementWeightedSignal | SignalType.TweetBasedUnifiedUniformSignal => convertSourceInfo(sourceType = SourceType.TweetAggregation, signals = ussSignals) case SignalType.ProducerBasedUnifiedEngagementWeightedSignal | SignalType.ProducerBasedUnifiedUniformSignal => convertSourceInfo(sourceType = SourceType.ProducerAggregation, signals = ussSignals) // Default case _ => Seq.empty[SourceInfo] } } sourceInfoList } } } override def convertSourceInfo( sourceType: SourceType, signals: Seq[SignalConvertType] ): Seq[SourceInfo] = { signals.map { signal => SourceInfo( sourceType = sourceType, internalId = signal.targetInternalId.getOrElse( throw new IllegalArgumentException( s"${sourceType.toString} Signal does not have internalId")), sourceEventTime = if (signal.timestamp == 0L) None else Some(Time.fromMilliseconds(signal.timestamp)) ) } } private def trackUssSignalStatsPerSignalType( query: FetcherQuery, signalType: SignalType, ussSignals: Seq[UssSignal] ): Unit = { val productScopedStats = stats.scope(query.product.originalName) val productUserStateScopedStats = productScopedStats.scope(query.userState.toString) val productStats = productScopedStats.scope(signalType.toString) val productUserStateStats = productUserStateScopedStats.scope(signalType.toString) productStats.counter(Success).incr() productUserStateStats.counter(Success).incr() val size = ussSignals.size productStats.stat(Size).add(size) productUserStateStats.stat(Size).add(size) if (size == 0) { productStats.counter(Empty).incr() productUserStateStats.counter(Empty).incr() } } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/UssStore.scala ================================================ package com.twitter.cr_mixer.source_signal import com.twitter.cr_mixer.param.GlobalParams import com.twitter.cr_mixer.param.GoodProfileClickParams import com.twitter.cr_mixer.param.GoodTweetClickParams import com.twitter.cr_mixer.param.RealGraphOonParams import com.twitter.cr_mixer.param.RecentFollowsParams import com.twitter.cr_mixer.param.RecentNegativeSignalParams import com.twitter.cr_mixer.param.RecentNotificationsParams import com.twitter.cr_mixer.param.RecentOriginalTweetsParams import com.twitter.cr_mixer.param.RecentReplyTweetsParams import com.twitter.cr_mixer.param.RecentRetweetsParams import com.twitter.cr_mixer.param.RecentTweetFavoritesParams import com.twitter.cr_mixer.param.RepeatedProfileVisitsParams import com.twitter.cr_mixer.param.TweetSharesParams import com.twitter.cr_mixer.param.UnifiedUSSSignalParams import com.twitter.cr_mixer.param.VideoViewTweetsParams import com.twitter.cr_mixer.source_signal.UssStore.Query import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.finagle.stats.StatsReceiver import com.twitter.simclusters_v2.common.UserId import com.twitter.storehaus.ReadableStore import com.twitter.usersignalservice.thriftscala.{Signal => UssSignal} import com.twitter.usersignalservice.thriftscala.SignalType import javax.inject.Singleton import com.twitter.timelines.configapi import com.twitter.timelines.configapi.Params import com.twitter.usersignalservice.thriftscala.BatchSignalRequest import com.twitter.usersignalservice.thriftscala.BatchSignalResponse import com.twitter.usersignalservice.thriftscala.SignalRequest import com.twitter.util.Future import com.twitter.cr_mixer.thriftscala.Product import com.twitter.usersignalservice.thriftscala.ClientIdentifier @Singleton case class UssStore( stratoStore: ReadableStore[BatchSignalRequest, BatchSignalResponse], statsReceiver: StatsReceiver) extends ReadableStore[Query, Seq[(SignalType, Seq[UssSignal])]] { import com.twitter.cr_mixer.source_signal.UssStore._ override def get(query: Query): Future[Option[Seq[(SignalType, Seq[UssSignal])]]] = { val ussClientIdentifier = query.product match { case Product.Home => ClientIdentifier.CrMixerHome case Product.Notifications => ClientIdentifier.CrMixerNotifications case Product.Email => ClientIdentifier.CrMixerEmail case _ => ClientIdentifier.Unknown } val batchSignalRequest = BatchSignalRequest( query.userId, buildUserSignalServiceRequests(query.params), Some(ussClientIdentifier)) stratoStore .get(batchSignalRequest) .map { _.map { batchSignalResponse => batchSignalResponse.signalResponse.toSeq.map { case (signalType, ussSignals) => (signalType, ussSignals) } } } } private def buildUserSignalServiceRequests( param: Params, ): Seq[SignalRequest] = { val unifiedMaxSourceKeyNum = param(GlobalParams.UnifiedMaxSourceKeyNum) val goodTweetClickMaxSignalNum = param(GoodTweetClickParams.MaxSignalNumParam) val aggrTweetMaxSourceKeyNum = param(UnifiedUSSSignalParams.UnifiedTweetSourceNumberParam) val aggrProducerMaxSourceKeyNum = param(UnifiedUSSSignalParams.UnifiedProducerSourceNumberParam) val maybeRecentTweetFavorite = if (param(RecentTweetFavoritesParams.EnableSourceParam)) Some(SignalRequest(Some(unifiedMaxSourceKeyNum), SignalType.TweetFavorite)) else None val maybeRecentRetweet = if (param(RecentRetweetsParams.EnableSourceParam)) Some(SignalRequest(Some(unifiedMaxSourceKeyNum), SignalType.Retweet)) else None val maybeRecentReply = if (param(RecentReplyTweetsParams.EnableSourceParam)) Some(SignalRequest(Some(unifiedMaxSourceKeyNum), SignalType.Reply)) else None val maybeRecentOriginalTweet = if (param(RecentOriginalTweetsParams.EnableSourceParam)) Some(SignalRequest(Some(unifiedMaxSourceKeyNum), SignalType.OriginalTweet)) else None val maybeRecentFollow = if (param(RecentFollowsParams.EnableSourceParam)) Some(SignalRequest(Some(unifiedMaxSourceKeyNum), SignalType.AccountFollow)) else None val maybeRepeatedProfileVisits = if (param(RepeatedProfileVisitsParams.EnableSourceParam)) Some( SignalRequest( Some(unifiedMaxSourceKeyNum), param(RepeatedProfileVisitsParams.ProfileMinVisitType).signalType)) else None val maybeRecentNotifications = if (param(RecentNotificationsParams.EnableSourceParam)) Some(SignalRequest(Some(unifiedMaxSourceKeyNum), SignalType.NotificationOpenAndClickV1)) else None val maybeTweetShares = if (param(TweetSharesParams.EnableSourceParam)) { Some(SignalRequest(Some(unifiedMaxSourceKeyNum), SignalType.TweetShareV1)) } else None val maybeRealGraphOon = if (param(RealGraphOonParams.EnableSourceParam)) { Some(SignalRequest(Some(unifiedMaxSourceKeyNum), SignalType.RealGraphOon)) } else None val maybeGoodTweetClick = if (param(GoodTweetClickParams.EnableSourceParam)) Some( SignalRequest( Some(goodTweetClickMaxSignalNum), param(GoodTweetClickParams.ClickMinDwellTimeType).signalType)) else None val maybeVideoViewTweets = if (param(VideoViewTweetsParams.EnableSourceParam)) { Some( SignalRequest( Some(unifiedMaxSourceKeyNum), param(VideoViewTweetsParams.VideoViewTweetTypeParam).signalType)) } else None val maybeGoodProfileClick = if (param(GoodProfileClickParams.EnableSourceParam)) Some( SignalRequest( Some(unifiedMaxSourceKeyNum), param(GoodProfileClickParams.ClickMinDwellTimeType).signalType)) else None val maybeAggTweetSignal = if (param(UnifiedUSSSignalParams.EnableTweetAggSourceParam)) Some( SignalRequest( Some(aggrTweetMaxSourceKeyNum), param(UnifiedUSSSignalParams.TweetAggTypeParam).signalType ) ) else None val maybeAggProducerSignal = if (param(UnifiedUSSSignalParams.EnableProducerAggSourceParam)) Some( SignalRequest( Some(aggrProducerMaxSourceKeyNum), param(UnifiedUSSSignalParams.ProducerAggTypeParam).signalType ) ) else None // negative signals val maybeNegativeSignals = if (param(RecentNegativeSignalParams.EnableSourceParam)) { EnabledNegativeSignalTypes .map(negativeSignal => SignalRequest(Some(unifiedMaxSourceKeyNum), negativeSignal)).toSeq } else Seq.empty val allPositiveSignals = if (param(UnifiedUSSSignalParams.ReplaceIndividualUSSSourcesParam)) Seq( maybeRecentOriginalTweet, maybeRecentNotifications, maybeRealGraphOon, maybeGoodTweetClick, maybeGoodProfileClick, maybeAggProducerSignal, maybeAggTweetSignal, ) else Seq( maybeRecentTweetFavorite, maybeRecentRetweet, maybeRecentReply, maybeRecentOriginalTweet, maybeRecentFollow, maybeRepeatedProfileVisits, maybeRecentNotifications, maybeTweetShares, maybeRealGraphOon, maybeGoodTweetClick, maybeVideoViewTweets, maybeGoodProfileClick, maybeAggProducerSignal, maybeAggTweetSignal, ) allPositiveSignals.flatten ++ maybeNegativeSignals } } object UssStore { case class Query( userId: UserId, params: configapi.Params, product: Product) val EnabledNegativeSourceTypes: Set[SourceType] = Set(SourceType.AccountBlock, SourceType.AccountMute) private val EnabledNegativeSignalTypes: Set[SignalType] = Set(SignalType.AccountBlock, SignalType.AccountMute) } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/storehaus:core", "3rdparty/jvm/org/lz4:lz4-java", "3rdparty/src/jvm/com/twitter/storehaus:core", "configapi/configapi-core", "content-recommender/thrift/src/main/thrift:thrift-scala", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/config", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", "cr-mixer/thrift/src/main/thrift:thrift-scala", "finatra/inject/inject-core/src/main/scala", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util:stats_util", "relevance-platform/src/main/scala/com/twitter/relevance_platform/common/stats", "src/java/com/twitter/search/common/schema/base", "src/java/com/twitter/search/common/schema/earlybird", "src/java/com/twitter/search/queryparser/query:core-query-nodes", "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", "src/scala/com/twitter/simclusters_v2/common", "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", "src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/search/common:ranking-scala", "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", ], ) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util/CandidateGenerationKeyUtil.scala ================================================ package com.twitter.cr_mixer.util import com.twitter.cr_mixer.model.CandidateGenerationInfo import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.thriftscala.CandidateGenerationKey import com.twitter.cr_mixer.thriftscala.SimilarityEngine import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.simclusters_v2.common.UserId import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.util.Time object CandidateGenerationKeyUtil { private val PlaceholderUserId = 0L // this default value will not be used private val DefaultSourceInfo: SourceInfo = SourceInfo( sourceType = SourceType.RequestUserId, sourceEventTime = None, internalId = InternalId.UserId(PlaceholderUserId) ) def toThrift( candidateGenerationInfo: CandidateGenerationInfo, requestUserId: UserId ): CandidateGenerationKey = { CandidateGenerationKey( sourceType = candidateGenerationInfo.sourceInfoOpt.getOrElse(DefaultSourceInfo).sourceType, sourceEventTime = candidateGenerationInfo.sourceInfoOpt .getOrElse(DefaultSourceInfo).sourceEventTime.getOrElse(Time.fromMilliseconds(0L)).inMillis, id = candidateGenerationInfo.sourceInfoOpt .map(_.internalId).getOrElse(InternalId.UserId(requestUserId)), modelId = candidateGenerationInfo.similarityEngineInfo.modelId.getOrElse(""), similarityEngineType = Some(candidateGenerationInfo.similarityEngineInfo.similarityEngineType), contributingSimilarityEngine = Some(candidateGenerationInfo.contributingSimilarityEngines.map(se => SimilarityEngine(se.similarityEngineType, se.modelId, se.score))) ) } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util/CountWeightedInterleaveUtil.scala ================================================ package com.twitter.cr_mixer.util import com.twitter.cr_mixer.model.Candidate import com.twitter.cr_mixer.model.InitialCandidate import com.twitter.cr_mixer.model.RankedCandidate import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.param.BlenderParams.BlendGroupingMethodEnum import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.simclusters_v2.thriftscala.InternalId object CountWeightedInterleaveUtil { /** * Grouping key for interleaving candidates * * @param sourceInfoOpt optional SourceInfo, containing the source information * @param similarityEngineTypeOpt optional SimilarityEngineType, containing similarity engine * information * @param modelIdOpt optional modelId, containing the model ID * @param authorIdOpt optional authorId, containing the tweet author ID * @param groupIdOpt optional groupId, containing the ID corresponding to the blending group */ case class GroupingKey( sourceInfoOpt: Option[SourceInfo], similarityEngineTypeOpt: Option[SimilarityEngineType], modelIdOpt: Option[String], authorIdOpt: Option[Long], groupIdOpt: Option[Int]) /** * Converts candidates to grouping key based upon the feature that we interleave with. */ def toGroupingKey[CandidateType <: Candidate]( candidate: CandidateType, interleaveFeature: Option[BlendGroupingMethodEnum.Value], groupId: Option[Int], ): GroupingKey = { val grouping: GroupingKey = candidate match { case c: RankedCandidate => interleaveFeature.getOrElse(BlendGroupingMethodEnum.SourceKeyDefault) match { case BlendGroupingMethodEnum.SourceKeyDefault => GroupingKey( sourceInfoOpt = c.reasonChosen.sourceInfoOpt, similarityEngineTypeOpt = Some(c.reasonChosen.similarityEngineInfo.similarityEngineType), modelIdOpt = c.reasonChosen.similarityEngineInfo.modelId, authorIdOpt = None, groupIdOpt = groupId ) // Some candidate sources don't have a sourceType, so it defaults to similarityEngine case BlendGroupingMethodEnum.SourceTypeSimilarityEngine => val sourceInfoOpt = c.reasonChosen.sourceInfoOpt.map(_.sourceType).map { sourceType => SourceInfo( sourceType = sourceType, internalId = InternalId.UserId(0), sourceEventTime = None) } GroupingKey( sourceInfoOpt = sourceInfoOpt, similarityEngineTypeOpt = Some(c.reasonChosen.similarityEngineInfo.similarityEngineType), modelIdOpt = c.reasonChosen.similarityEngineInfo.modelId, authorIdOpt = None, groupIdOpt = groupId ) case BlendGroupingMethodEnum.AuthorId => GroupingKey( sourceInfoOpt = None, similarityEngineTypeOpt = None, modelIdOpt = None, authorIdOpt = Some(c.tweetInfo.authorId), groupIdOpt = groupId ) case _ => throw new UnsupportedOperationException( s"Unsupported interleave feature: $interleaveFeature") } case _ => GroupingKey( sourceInfoOpt = None, similarityEngineTypeOpt = None, modelIdOpt = None, authorIdOpt = None, groupIdOpt = groupId ) } grouping } /** * Rather than manually calculating and maintaining the weights to rank with, we instead * calculate the weights on the fly, based upon the frequencies of the candidates within each * group. To ensure that diversity of the feature is maintained, we additionally employ a * 'shrinkage' parameter which enforces more diversity by moving the weights closer to uniformity. * More details are available at go/weighted-interleave. * * @param candidateSeqKeyByFeature candidate to key. * @param rankerWeightShrinkage value between [0, 1] with 1 being complete uniformity. * @return Interleaving weights keyed by feature. */ private def calculateWeightsKeyByFeature[CandidateType <: Candidate]( candidateSeqKeyByFeature: Map[GroupingKey, Seq[CandidateType]], rankerWeightShrinkage: Double ): Map[GroupingKey, Double] = { val maxNumberCandidates: Double = candidateSeqKeyByFeature.values .map { candidates => candidates.size }.max.toDouble candidateSeqKeyByFeature.map { case (featureKey: GroupingKey, candidateSeq: Seq[CandidateType]) => val observedWeight: Double = candidateSeq.size.toDouble / maxNumberCandidates // How much to shrink empirical estimates to 1 (Default is to make all weights 1). val finalWeight = (1.0 - rankerWeightShrinkage) * observedWeight + rankerWeightShrinkage * 1.0 featureKey -> finalWeight } } /** * Builds out the groups and weights for weighted interleaving of the candidates. * More details are available at go/weighted-interleave. * * @param rankedCandidateSeq candidates to interleave. * @param rankerWeightShrinkage value between [0, 1] with 1 being complete uniformity. * @return Candidates grouped by feature key and with calculated interleaving weights. */ def buildRankedCandidatesWithWeightKeyByFeature( rankedCandidateSeq: Seq[RankedCandidate], rankerWeightShrinkage: Double, interleaveFeature: BlendGroupingMethodEnum.Value ): Seq[(Seq[RankedCandidate], Double)] = { // To accommodate the re-grouping in InterleaveRanker // In InterleaveBlender, we have already abandoned the grouping keys, and use Seq[Seq[]] to do interleave // Since that we build the candidateSeq with groupingKey, we can guarantee there is no empty candidateSeq val candidateSeqKeyByFeature: Map[GroupingKey, Seq[RankedCandidate]] = rankedCandidateSeq.groupBy { candidate: RankedCandidate => toGroupingKey(candidate, Some(interleaveFeature), None) } // These weights [0, 1] are used to do weighted interleaving // The default value of 1.0 ensures the group is always sampled. val candidateWeightsKeyByFeature: Map[GroupingKey, Double] = calculateWeightsKeyByFeature(candidateSeqKeyByFeature, rankerWeightShrinkage) candidateSeqKeyByFeature.map { case (groupingKey: GroupingKey, candidateSeq: Seq[RankedCandidate]) => Tuple2( candidateSeq.sortBy(-_.predictionScore), candidateWeightsKeyByFeature.getOrElse(groupingKey, 1.0)) }.toSeq } /** * Takes current grouping (as implied by the outer Seq) and computes blending weights. * * @param initialCandidatesSeqSeq grouped candidates to interleave. * @param rankerWeightShrinkage value between [0, 1] with 1 being complete uniformity. * @return Grouped candidates with calculated interleaving weights. */ def buildInitialCandidatesWithWeightKeyByFeature( initialCandidatesSeqSeq: Seq[Seq[InitialCandidate]], rankerWeightShrinkage: Double, ): Seq[(Seq[InitialCandidate], Double)] = { val candidateSeqKeyByFeature: Map[GroupingKey, Seq[InitialCandidate]] = initialCandidatesSeqSeq.zipWithIndex.map(_.swap).toMap.map { case (groupId: Int, initialCandidatesSeq: Seq[InitialCandidate]) => toGroupingKey(initialCandidatesSeq.head, None, Some(groupId)) -> initialCandidatesSeq } // These weights [0, 1] are used to do weighted interleaving // The default value of 1.0 ensures the group is always sampled. val candidateWeightsKeyByFeature = calculateWeightsKeyByFeature(candidateSeqKeyByFeature, rankerWeightShrinkage) candidateSeqKeyByFeature.map { case (groupingKey: GroupingKey, candidateSeq: Seq[InitialCandidate]) => Tuple2(candidateSeq, candidateWeightsKeyByFeature.getOrElse(groupingKey, 1.0)) }.toSeq } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util/EarlybirdSearchUtil.scala ================================================ package com.twitter.cr_mixer.util import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant import com.twitter.search.queryparser.query.search.SearchOperator import com.twitter.search.queryparser.query.search.SearchOperatorConstants import com.twitter.search.queryparser.query.{Query => EbQuery} import com.twitter.search.queryparser.query.Conjunction import scala.collection.JavaConverters._ import com.twitter.search.earlybird.thriftscala.ThriftSearchResultMetadataOptions import com.twitter.simclusters_v2.common.TweetId import com.twitter.search.queryparser.query.Query import com.twitter.util.Duration import com.twitter.search.common.query.thriftjava.thriftscala.CollectorTerminationParams object EarlybirdSearchUtil { val EarlybirdClientId: String = "cr-mixer.prod" val Mentions: String = EarlybirdFieldConstant.MENTIONS_FACET val Hashtags: String = EarlybirdFieldConstant.HASHTAGS_FACET val FacetsToFetch: Seq[String] = Seq(Mentions, Hashtags) val MetadataOptions: ThriftSearchResultMetadataOptions = ThriftSearchResultMetadataOptions( getTweetUrls = true, getResultLocation = false, getLuceneScore = false, getInReplyToStatusId = true, getReferencedTweetAuthorId = true, getMediaBits = true, getAllFeatures = true, getFromUserId = true, returnSearchResultFeatures = true, // Set getExclusiveConversationAuthorId in order to retrieve Exclusive / SuperFollow tweets. getExclusiveConversationAuthorId = true ) // Filter out retweets and replies val TweetTypesToExclude: Seq[String] = Seq( SearchOperatorConstants.NATIVE_RETWEETS, SearchOperatorConstants.REPLIES) def GetCollectorTerminationParams( maxNumHitsPerShard: Int, processingTimeout: Duration ): Option[CollectorTerminationParams] = { Some( CollectorTerminationParams( // maxHitsToProcess is used for early termination on each EB shard maxHitsToProcess = Some(maxNumHitsPerShard), timeoutMs = processingTimeout.inMilliseconds.toInt )) } /** * Get EarlybirdQuery * This function creates a EBQuery based on the search input */ def GetEarlybirdQuery( beforeTweetIdExclusive: Option[TweetId], afterTweetIdExclusive: Option[TweetId], excludedTweetIds: Set[TweetId], filterOutRetweetsAndReplies: Boolean ): Option[EbQuery] = CreateConjunction( Seq( CreateRangeQuery(beforeTweetIdExclusive, afterTweetIdExclusive), CreateExcludedTweetIdsQuery(excludedTweetIds), CreateTweetTypesFilters(filterOutRetweetsAndReplies) ).flatten) def CreateRangeQuery( beforeTweetIdExclusive: Option[TweetId], afterTweetIdExclusive: Option[TweetId] ): Option[EbQuery] = { val beforeIdClause = beforeTweetIdExclusive.map { beforeId => // MAX_ID is an inclusive value therefore we subtract 1 from beforeId. new SearchOperator(SearchOperator.Type.MAX_ID, (beforeId - 1).toString) } val afterIdClause = afterTweetIdExclusive.map { afterId => new SearchOperator(SearchOperator.Type.SINCE_ID, afterId.toString) } CreateConjunction(Seq(beforeIdClause, afterIdClause).flatten) } def CreateTweetTypesFilters(filterOutRetweetsAndReplies: Boolean): Option[EbQuery] = { if (filterOutRetweetsAndReplies) { val tweetTypeFilters = TweetTypesToExclude.map { searchOperator => new SearchOperator(SearchOperator.Type.EXCLUDE, searchOperator) } CreateConjunction(tweetTypeFilters) } else None } def CreateConjunction(clauses: Seq[EbQuery]): Option[EbQuery] = { clauses.size match { case 0 => None case 1 => Some(clauses.head) case _ => Some(new Conjunction(clauses.asJava)) } } def CreateExcludedTweetIdsQuery(tweetIds: Set[TweetId]): Option[EbQuery] = { if (tweetIds.nonEmpty) { Some( new SearchOperator.Builder() .setType(SearchOperator.Type.NAMED_MULTI_TERM_DISJUNCTION) .addOperand(EarlybirdFieldConstant.ID_FIELD.getFieldName) .addOperand(EXCLUDE_TWEET_IDS) .setOccur(Query.Occur.MUST_NOT) .build()) } else None } /** * Get NamedDisjunctions with excludedTweetIds */ def GetNamedDisjunctions(excludedTweetIds: Set[TweetId]): Option[Map[String, Seq[Long]]] = if (excludedTweetIds.nonEmpty) createNamedDisjunctionsExcludedTweetIds(excludedTweetIds) else None val EXCLUDE_TWEET_IDS = "exclude_tweet_ids" private def createNamedDisjunctionsExcludedTweetIds( tweetIds: Set[TweetId] ): Option[Map[String, Seq[Long]]] = { if (tweetIds.nonEmpty) { Some(Map(EXCLUDE_TWEET_IDS -> tweetIds.toSeq)) } else None } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util/InterleaveUtil.scala ================================================ package com.twitter.cr_mixer.util import com.twitter.cr_mixer.model.Candidate import com.twitter.cr_mixer.model.CandidateGenerationInfo import com.twitter.cr_mixer.model.RankedCandidate import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.simclusters_v2.common.TweetId import scala.collection.mutable import scala.collection.mutable.ArrayBuffer object InterleaveUtil { /** * Interleaves candidates by iteratively taking one candidate from the 1st Seq and adding it to the result. * Once we take a candidate from a Seq, we move this Seq to the end of the queue to process, * and remove the candidate from that Seq. * * We keep a mutable.Set[TweetId] buffer to ensure there are no duplicates. * * @param candidates candidates assumed to be sorted by eventTime (latest event comes first) * @return interleaved candidates */ def interleave[CandidateType <: Candidate]( candidates: Seq[Seq[CandidateType]] ): Seq[CandidateType] = { // copy candidates into a mutable map so this method is thread-safe val candidatesPerSequence = candidates.map { tweetCandidates => mutable.Queue() ++= tweetCandidates } val seen = mutable.Set[TweetId]() val candidateSeqQueue = mutable.Queue() ++= candidatesPerSequence val result = ArrayBuffer[CandidateType]() while (candidateSeqQueue.nonEmpty) { val candidatesQueue = candidateSeqQueue.head if (candidatesQueue.nonEmpty) { val candidate = candidatesQueue.dequeue() val candidateTweetId = candidate.tweetId val seenCandidate = seen.contains(candidateTweetId) if (!seenCandidate) { result += candidate seen.add(candidate.tweetId) candidateSeqQueue.enqueue( candidateSeqQueue.dequeue() ) // move this Seq to end } } else { candidateSeqQueue.dequeue() //finished processing this Seq } } //convert result to immutable seq result.toList } /** * Interleaves candidates by iteratively * 1. Checking weight to see if enough accumulation has occurred to sample from * 2. If yes, taking one candidate from the the Seq and adding it to the result. * 3. Move this Seq to the end of the queue to process (and remove the candidate from that Seq if * we sampled it from step 2). * * We keep count of the iterations to prevent infinite loops. * We keep a mutable.Set[TweetId] buffer to ensure there are no duplicates. * * @param candidatesAndWeight candidates assumed to be sorted by eventTime (latest event comes first), * along with sampling weights to help prioritize important groups. * @param maxWeightAdjustments Maximum number of iterations to account for weighting before * defaulting to uniform interleaving. * @return interleaved candidates */ def weightedInterleave[CandidateType <: Candidate]( candidatesAndWeight: Seq[(Seq[CandidateType], Double)], maxWeightAdjustments: Int = 0 ): Seq[CandidateType] = { // Set to avoid numerical issues around 1.0 val min_weight = 1 - 1e-30 // copy candidates into a mutable map so this method is thread-safe // adds a counter to use towards sampling val candidatesAndWeightsPerSequence: Seq[ (mutable.Queue[CandidateType], InterleaveWeights) ] = candidatesAndWeight.map { candidatesAndWeight => (mutable.Queue() ++= candidatesAndWeight._1, InterleaveWeights(candidatesAndWeight._2, 0.0)) } val seen: mutable.Set[TweetId] = mutable.Set[TweetId]() val candidateSeqQueue: mutable.Queue[(mutable.Queue[CandidateType], InterleaveWeights)] = mutable.Queue() ++= candidatesAndWeightsPerSequence val result: ArrayBuffer[CandidateType] = ArrayBuffer[CandidateType]() var number_iterations: Int = 0 while (candidateSeqQueue.nonEmpty) { val (candidatesQueue, currentWeights) = candidateSeqQueue.head if (candidatesQueue.nonEmpty) { // Confirm weighting scheme currentWeights.summed_weight += currentWeights.weight number_iterations += 1 if (currentWeights.summed_weight >= min_weight || number_iterations >= maxWeightAdjustments) { // If we sample, then adjust the counter currentWeights.summed_weight -= 1.0 val candidate = candidatesQueue.dequeue() val candidateTweetId = candidate.tweetId val seenCandidate = seen.contains(candidateTweetId) if (!seenCandidate) { result += candidate seen.add(candidate.tweetId) candidateSeqQueue.enqueue(candidateSeqQueue.dequeue()) // move this Seq to end } } else { candidateSeqQueue.enqueue(candidateSeqQueue.dequeue()) // move this Seq to end } } else { candidateSeqQueue.dequeue() //finished processing this Seq } } //convert result to immutable seq result.toList } def buildCandidatesKeyByCGInfo( candidates: Seq[RankedCandidate], ): Seq[Seq[RankedCandidate]] = { // To accommodate the re-grouping in InterleaveRanker // In InterleaveBlender, we have already abandoned the grouping keys, and use Seq[Seq[]] to do interleave // Since that we build the candidateSeq with groupingKey, we can guarantee there is no empty candidateSeq val candidateSeqKeyByCG = candidates.groupBy(candidate => GroupingKey.toGroupingKey(candidate.reasonChosen)) candidateSeqKeyByCG.map { case (groupingKey, candidateSeq) => candidateSeq.sortBy(-_.predictionScore) }.toSeq } } case class GroupingKey( sourceInfoOpt: Option[SourceInfo], similarityEngineType: SimilarityEngineType, modelId: Option[String]) {} object GroupingKey { def toGroupingKey(candidateGenerationInfo: CandidateGenerationInfo): GroupingKey = { GroupingKey( candidateGenerationInfo.sourceInfoOpt, candidateGenerationInfo.similarityEngineInfo.similarityEngineType, candidateGenerationInfo.similarityEngineInfo.modelId ) } } case class InterleaveWeights(weight: Double, var summed_weight: Double) ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util/MetricTagUtil.scala ================================================ package com.twitter.cr_mixer.util import com.twitter.cr_mixer.model.RankedCandidate import com.twitter.cr_mixer.model.SimilarityEngineInfo import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.thriftscala.MetricTag import com.twitter.cr_mixer.thriftscala.SimilarityEngineType import com.twitter.cr_mixer.thriftscala.SourceType object MetricTagUtil { def buildMetricTags(candidate: RankedCandidate): Seq[MetricTag] = { val interestedInMetricTag = isFromInterestedIn(candidate) val cgInfoMetricTags = candidate.potentialReasons .flatMap { cgInfo => val sourceMetricTag = cgInfo.sourceInfoOpt.flatMap { sourceInfo => toMetricTagFromSource(sourceInfo.sourceType) } val similarityEngineTags = toMetricTagFromSimilarityEngine( cgInfo.similarityEngineInfo, cgInfo.contributingSimilarityEngines) val combinedMetricTag = cgInfo.sourceInfoOpt.flatMap { sourceInfo => toMetricTagFromSourceAndSimilarityEngine(sourceInfo, cgInfo.similarityEngineInfo) } Seq(sourceMetricTag) ++ similarityEngineTags ++ Seq(combinedMetricTag) }.flatten.toSet (interestedInMetricTag ++ cgInfoMetricTags).toSeq } /*** * match a sourceType to a metricTag */ private def toMetricTagFromSource(sourceType: SourceType): Option[MetricTag] = { sourceType match { case SourceType.TweetFavorite => Some(MetricTag.TweetFavorite) // Personalized Topics in Home case SourceType.Retweet => Some(MetricTag.Retweet) // Personalized Topics in Home case SourceType.NotificationClick => Some(MetricTag.PushOpenOrNtabClick) // Health Filter in MR case SourceType.OriginalTweet => Some(MetricTag.OriginalTweet) case SourceType.Reply => Some(MetricTag.Reply) case SourceType.TweetShare => Some(MetricTag.TweetShare) case SourceType.UserFollow => Some(MetricTag.UserFollow) case SourceType.UserRepeatedProfileVisit => Some(MetricTag.UserRepeatedProfileVisit) case SourceType.TwiceUserId => Some(MetricTag.TwiceUserId) case _ => None } } /*** * If the SEInfo is built by a unified sim engine, we un-wrap the contributing sim engines. * If not, we log the sim engine as usual. * @param seInfo (CandidateGenerationInfo.similarityEngineInfo): SimilarityEngineInfo, * @param cseInfo (CandidateGenerationInfo.contributingSimilarityEngines): Seq[SimilarityEngineInfo] */ private def toMetricTagFromSimilarityEngine( seInfo: SimilarityEngineInfo, cseInfo: Seq[SimilarityEngineInfo] ): Seq[Option[MetricTag]] = { seInfo.similarityEngineType match { case SimilarityEngineType.TweetBasedUnifiedSimilarityEngine => // un-wrap the unified sim engine cseInfo.map { contributingSimEngine => toMetricTagFromSimilarityEngine(contributingSimEngine, Seq.empty) }.flatten case SimilarityEngineType.ProducerBasedUnifiedSimilarityEngine => // un-wrap the unified sim engine cseInfo.map { contributingSimEngine => toMetricTagFromSimilarityEngine(contributingSimEngine, Seq.empty) }.flatten // SimClustersANN can either be called on its own, or be called under unified sim engine case SimilarityEngineType.SimClustersANN => // the old "UserInterestedIn" will be replaced by this. Also, OfflineTwice Seq(Some(MetricTag.SimClustersANN), seInfo.modelId.flatMap(toMetricTagFromModelId(_))) case SimilarityEngineType.ConsumerEmbeddingBasedTwHINANN => Seq(Some(MetricTag.ConsumerEmbeddingBasedTwHINANN)) case SimilarityEngineType.TwhinCollabFilter => Seq(Some(MetricTag.TwhinCollabFilter)) // In the current implementation, TweetBasedUserTweetGraph/TweetBasedTwHINANN has a tag when // it's either a base SE or a contributing SE. But for now they only show up in contributing SE. case SimilarityEngineType.TweetBasedUserTweetGraph => Seq(Some(MetricTag.TweetBasedUserTweetGraph)) case SimilarityEngineType.TweetBasedTwHINANN => Seq(Some(MetricTag.TweetBasedTwHINANN)) case _ => Seq.empty } } /*** * pass in a model id, and match it with the metric tag type. */ private def toMetricTagFromModelId( modelId: String ): Option[MetricTag] = { val pushOpenBasedModelRegex = "(.*_Model20m145k2020_20220819)".r modelId match { case pushOpenBasedModelRegex(_*) => Some(MetricTag.RequestHealthFilterPushOpenBasedTweetEmbedding) case _ => None } } private def toMetricTagFromSourceAndSimilarityEngine( sourceInfo: SourceInfo, seInfo: SimilarityEngineInfo ): Option[MetricTag] = { sourceInfo.sourceType match { case SourceType.Lookalike if seInfo.similarityEngineType == SimilarityEngineType.ConsumersBasedUserTweetGraph => Some(MetricTag.LookalikeUTG) case _ => None } } /** * Special use case: used by Notifications team to generate the UserInterestedIn CRT push copy. * * if we have different types of InterestedIn (eg. UserInterestedIn, NextInterestedIn), * this if statement will have to be refactored to contain the real UserInterestedIn. * @return */ private def isFromInterestedIn(candidate: RankedCandidate): Set[MetricTag] = { if (candidate.reasonChosen.sourceInfoOpt.isEmpty && candidate.reasonChosen.similarityEngineInfo.similarityEngineType == SimilarityEngineType.SimClustersANN) { Set(MetricTag.UserInterestedIn) } else Set.empty } } ================================================ FILE: cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util/SignalTimestampStatsUtil.scala ================================================ package com.twitter.cr_mixer.util import com.twitter.cr_mixer.model.CandidateGenerationInfo import com.twitter.cr_mixer.model.RankedCandidate import com.twitter.cr_mixer.model.SourceInfo import com.twitter.cr_mixer.thriftscala.SourceType import com.twitter.cr_mixer.thriftscala.TweetRecommendation import javax.inject.Inject import com.twitter.finagle.stats.StatsReceiver import javax.inject.Singleton import com.twitter.relevance_platform.common.stats.BucketTimestampStats @Singleton class SignalTimestampStatsUtil @Inject() (statsReceiver: StatsReceiver) { import SignalTimestampStatsUtil._ private val signalDelayAgePerDayStats = new BucketTimestampStats[TweetRecommendation]( BucketTimestampStats.MillisecondsPerDay, _.latestSourceSignalTimestampInMillis.getOrElse(0), Some(SignalTimestampMaxDays))( statsReceiver.scope("signal_timestamp_per_day") ) // only stats past 90 days private val signalDelayAgePerHourStats = new BucketTimestampStats[TweetRecommendation]( BucketTimestampStats.MillisecondsPerHour, _.latestSourceSignalTimestampInMillis.getOrElse(0), Some(SignalTimestampMaxHours))( statsReceiver.scope("signal_timestamp_per_hour") ) // only stats past 24 hours private val signalDelayAgePerMinStats = new BucketTimestampStats[TweetRecommendation]( BucketTimestampStats.MillisecondsPerMinute, _.latestSourceSignalTimestampInMillis.getOrElse(0), Some(SignalTimestampMaxMins))( statsReceiver.scope("signal_timestamp_per_min") ) // only stats past 60 minutes def statsSignalTimestamp( tweets: Seq[TweetRecommendation], ): Seq[TweetRecommendation] = { signalDelayAgePerMinStats.count(tweets) signalDelayAgePerHourStats.count(tweets) signalDelayAgePerDayStats.count(tweets) } } object SignalTimestampStatsUtil { val SignalTimestampMaxMins = 60 // stats at most 60 mins val SignalTimestampMaxHours = 24 // stats at most 24 hours val SignalTimestampMaxDays = 90 // stats at most 90 days def buildLatestSourceSignalTimestamp(candidate: RankedCandidate): Option[Long] = { val timestampSeq = candidate.potentialReasons .collect { case CandidateGenerationInfo(Some(SourceInfo(sourceType, _, Some(sourceEventTime))), _, _) if sourceType == SourceType.TweetFavorite => sourceEventTime.inMilliseconds } if (timestampSeq.nonEmpty) { Some(timestampSeq.max(Ordering.Long)) } else { None } } } ================================================ FILE: cr-mixer/thrift/src/main/thrift/BUILD ================================================ create_thrift_libraries( base_name = "thrift", sources = ["*.thrift"], platform = "java8", tags = ["bazel-compatible"], dependency_roots = [ "finatra-internal/thrift/src/main/thrift", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift", "src/thrift/com/twitter/ads/schema:common", "src/thrift/com/twitter/ml/api:data", "src/thrift/com/twitter/recos:recos-common", "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift", "src/thrift/com/twitter/timelines/render:thrift", "strato/config/src/thrift/com/twitter/strato/graphql", "strato/config/src/thrift/com/twitter/strato/graphql:api-media-graphql", "strato/config/src/thrift/com/twitter/strato/graphql:topics-graphql", ], generate_languages = [ "java", "scala", "strato", ], provides_java_name = "cr-mixer-thrift-java", provides_scala_name = "cr-mixer-thrift-scala", ) create_thrift_libraries( base_name = "cr-mixer-scribe", sources = ["*.thrift"], tags = ["bazel-compatible"], dependency_roots = [ "finatra-internal/thrift/src/main/thrift", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift", "src/thrift/com/twitter/ads/schema:common", "src/thrift/com/twitter/ml/api:data", "src/thrift/com/twitter/recos:recos-common", "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift", "src/thrift/com/twitter/timelines/render:thrift", "strato/config/src/thrift/com/twitter/strato/graphql", ], generate_languages = [ "java", "scala", "strato", ], provides_java_name = "cr-mixer-scribe-java", provides_scala_name = "cr-mixer-scribe-scala", ) ================================================ FILE: cr-mixer/thrift/src/main/thrift/ads.thrift ================================================ namespace java com.twitter.cr_mixer.thriftjava #@namespace scala com.twitter.cr_mixer.thriftscala #@namespace strato com.twitter.cr_mixer include "product.thrift" include "product_context.thrift" include "com/twitter/product_mixer/core/client_context.thrift" include "com/twitter/ads/schema/shared.thrift" struct AdsRequest { 1: required client_context.ClientContext clientContext 2: required product.Product product # Product-specific parameters should be placed in the Product Context 3: optional product_context.ProductContext productContext 4: optional list excludedTweetIds (personalDataType = 'TweetId') } (persisted='true', hasPersonalData='true') struct AdsResponse { 1: required list ads } (persisted='true') struct AdTweetRecommendation { 1: required i64 tweetId (personalDataType = 'TweetId') 2: required double score 3: optional list lineItems } (persisted='true') struct LineItemInfo { 1: required i64 lineItemId (personalDataType = 'LineItemId') 2: required shared.LineItemObjective lineItemObjective } (persisted='true') ================================================ FILE: cr-mixer/thrift/src/main/thrift/candidate_generation_key.thrift ================================================ namespace java com.twitter.cr_mixer.thriftjava #@namespace scala com.twitter.cr_mixer.thriftscala #@namespace strato com.twitter.cr_mixer include "source_type.thrift" include "com/twitter/simclusters_v2/identifier.thrift" struct SimilarityEngine { 1: required source_type.SimilarityEngineType similarityEngineType 2: optional string modelId 3: optional double score } (persisted='true') struct CandidateGenerationKey { 1: required source_type.SourceType sourceType 2: required i64 sourceEventTime (personalDataType = 'PrivateTimestamp') 3: required identifier.InternalId id 4: required string modelId 5: optional source_type.SimilarityEngineType similarityEngineType 6: optional list contributingSimilarityEngine } (persisted='true') ================================================ FILE: cr-mixer/thrift/src/main/thrift/cr_mixer.thrift ================================================ namespace java com.twitter.cr_mixer.thriftjava #@namespace scala com.twitter.cr_mixer.thriftscala #@namespace strato com.twitter.cr_mixer include "ads.thrift" include "candidate_generation_key.thrift" include "product.thrift" include "product_context.thrift" include "validation.thrift" include "metric_tags.thrift" include "related_tweet.thrift" include "uteg.thrift" include "frs_based_tweet.thrift" include "related_video_tweet.thrift" include "topic_tweet.thrift" include "com/twitter/product_mixer/core/client_context.thrift" include "com/twitter/timelines/render/response.thrift" include "finatra-thrift/finatra_thrift_exceptions.thrift" include "com/twitter/strato/graphql/slice.thrift" struct CrMixerTweetRequest { 1: required client_context.ClientContext clientContext 2: required product.Product product # Product-specific parameters should be placed in the Product Context 3: optional product_context.ProductContext productContext 4: optional list excludedTweetIds (personalDataType = 'TweetId') } (persisted='true', hasPersonalData='true') struct TweetRecommendation { 1: required i64 tweetId (personalDataType = 'TweetId') 2: required double score 3: optional list metricTags # 4: the author of the tweet candidate. To be used by Content-Mixer to unblock the Hydra experiment. 4: optional i64 authorId (personalDataType = 'UserId') # 5: extra info about candidate generation. To be used by Content-Mixer to unblock the Hydra experiment. 5: optional candidate_generation_key.CandidateGenerationKey candidateGenerationKey # 1001: the latest timestamp of fav signals. If null, the candidate is not generated from fav signals 1001: optional i64 latestSourceSignalTimestampInMillis(personalDataType = 'PublicTimestamp') } (persisted='true', hasPersonalData = 'true') struct CrMixerTweetResponse { 1: required list tweets } (persisted='true') service CrMixer { CrMixerTweetResponse getTweetRecommendations(1: CrMixerTweetRequest request) throws ( # Validation errors - the details of which will be reported to clients on failure 1: validation.ValidationExceptionList validationErrors; # Server errors - the details of which will not be reported to clients 2: finatra_thrift_exceptions.ServerError serverError ) # getRelatedTweetsForQueryTweet and getRelatedTweetsForQueryAuthor do very similar things # We can merge these two endpoints into one unified endpoint related_tweet.RelatedTweetResponse getRelatedTweetsForQueryTweet(1: related_tweet.RelatedTweetRequest request) throws ( # Validation errors - the details of which will be reported to clients on failure 1: validation.ValidationExceptionList validationErrors; # Server errors - the details of which will not be reported to clients 2: finatra_thrift_exceptions.ServerError serverError ) related_tweet.RelatedTweetResponse getRelatedTweetsForQueryAuthor(1: related_tweet.RelatedTweetRequest request) throws ( # Validation errors - the details of which will be reported to clients on failure 1: validation.ValidationExceptionList validationErrors; # Server errors - the details of which will not be reported to clients 2: finatra_thrift_exceptions.ServerError serverError ) uteg.UtegTweetResponse getUtegTweetRecommendations(1: uteg.UtegTweetRequest request) throws ( # Validation errors - the details of which will be reported to clients on failure 1: validation.ValidationExceptionList validationErrors; # Server errors - the details of which will not be reported to clients 2: finatra_thrift_exceptions.ServerError serverError ) frs_based_tweet.FrsTweetResponse getFrsBasedTweetRecommendations(1: frs_based_tweet.FrsTweetRequest request) throws ( # Validation errors - the details of which will be reported to clients on failure 1: validation.ValidationExceptionList validationErrors; # Server errors - the details of which will not be reported to clients 2: finatra_thrift_exceptions.ServerError serverError ) related_video_tweet.RelatedVideoTweetResponse getRelatedVideoTweetsForQueryTweet(1: related_video_tweet.RelatedVideoTweetRequest request) throws ( # Validation errors - the details of which will be reported to clients on failure 1: validation.ValidationExceptionList validationErrors; # Server errors - the details of which will not be reported to clients 2: finatra_thrift_exceptions.ServerError serverError ) ads.AdsResponse getAdsRecommendations(1: ads.AdsRequest request) throws ( # Validation errors - the details of which will be reported to clients on failure 1: validation.ValidationExceptionList validationErrors; # Server errors - the details of which will not be reported to clients 2: finatra_thrift_exceptions.ServerError serverError ) topic_tweet.TopicTweetResponse getTopicTweetRecommendations(1: topic_tweet.TopicTweetRequest request) throws ( # Validation errors - the details of which will be reported to clients on failure 1: validation.ValidationExceptionList validationErrors; # Server errors - the details of which will not be reported to clients 2: finatra_thrift_exceptions.ServerError serverError ) } ================================================ FILE: cr-mixer/thrift/src/main/thrift/frs_based_tweet.thrift ================================================ namespace java com.twitter.cr_mixer.thriftjava #@namespace scala com.twitter.cr_mixer.thriftscala #@namespace strato com.twitter.cr_mixer include "product.thrift" include "product_context.thrift" include "com/twitter/product_mixer/core/client_context.thrift" struct FrsTweetRequest { 1: required client_context.ClientContext clientContext 2: required product.Product product 3: optional product_context.ProductContext productContext # excludedUserIds - user ids to be excluded from FRS candidate generation 4: optional list excludedUserIds (personalDataType = 'UserId') # excludedTweetIds - tweet ids to be excluded from Earlybird candidate generation 5: optional list excludedTweetIds (personalDataType = 'TweetId') } (persisted='true', hasPersonalData='true') struct FrsTweet { 1: required i64 tweetId (personalDataType = 'TweetId') 2: required i64 authorId (personalDataType = 'UserId') # skip 3 in case we need tweet score in the future # frsPrimarySource - which FRS candidate source is the primary one to generate this author 4: optional i32 frsPrimarySource # frsCandidateSourceScores - FRS candidate sources and the scores for this author # for i32 to algorithm mapping, see https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/hermit/hermit-core/src/main/scala/com/twitter/hermit/constants/AlgorithmFeedbackTokens.scala?L12 5: optional map frsCandidateSourceScores # frsPrimaryScore - the score of the FRS primary candidate source 6: optional double frsAuthorScore } (persisted='true', hasPersonalData = 'true') struct FrsTweetResponse { 1: required list tweets } (persisted='true') ================================================ FILE: cr-mixer/thrift/src/main/thrift/metric_tags.thrift ================================================ namespace java com.twitter.cr_mixer.thriftjava #@namespace scala com.twitter.cr_mixer.thriftscala #@namespace strato com.twitter.cr_mixer // NOTE: DO NOT depend on MetricTags for important ML Features or business logic. // MetricTags are meant for stats tracking & debugging purposes ONLY. // cr-mixer may change its definitions & how each candidate is tagged without public notice. // NOTE: TSPS needs the caller (Home) to specify which signal it uses to make Personalized Topics enum MetricTag { // Source Signal Tags TweetFavorite = 0 Retweet = 1 TrafficAttribution = 2 OriginalTweet = 3 Reply = 4 TweetShare = 5 UserFollow = 101 UserRepeatedProfileVisit = 102 PushOpenOrNtabClick = 201 HomeTweetClick = 301 HomeVideoView = 302 // sim engine types SimClustersANN = 401 TweetBasedUserTweetGraph = 402 TweetBasedTwHINANN = 403 ConsumerEmbeddingBasedTwHINANN = 404 // combined engine types UserInterestedIn = 501 // Will deprecate soon LookalikeUTG = 502 TwhinCollabFilter = 503 // Offline Twice TwiceUserId = 601 // Other Metric Tags RequestHealthFilterPushOpenBasedTweetEmbedding = 701 } (persisted='true', hasPersonalData='true') ================================================ FILE: cr-mixer/thrift/src/main/thrift/product.thrift ================================================ namespace java com.twitter.cr_mixer.thriftjava #@namespace scala com.twitter.cr_mixer.thriftscala #@namespace strato com.twitter.cr_mixer # In CrMixer, one org should only have one Product enum Product { Home = 1 Notifications = 2 Email = 3 MoreTweetsModule = 4 # aka RUX ImmersiveMediaViewer = 5 VideoCarousel = 6 ExploreTopics = 7 Ads = 8 HomeRealTime = 9 // Home Real-Time Tab is considered as a different Product surface to Home Tab. It's in early experiment phase. TopicLandingPage = 10 HomeTopicsBackfill = 11 TopicTweetsStrato = 12 } ================================================ FILE: cr-mixer/thrift/src/main/thrift/product_context.thrift ================================================ namespace java com.twitter.cr_mixer.thriftjava #@namespace scala com.twitter.cr_mixer.thriftscala #@namespace strato com.twitter.cr_mixer struct HomeContext { 2: optional i32 maxResults // enabled for QuaityFactor related DDGs only } (persisted='true', hasPersonalData='false') struct NotificationsContext { 1: optional i32 devNull // not being used. it's a placeholder } (persisted='true', hasPersonalData='false') struct ExploreContext { 1: required bool isVideoOnly } (persisted='true', hasPersonalData='false') union ProductContext { 1: HomeContext homeContext 2: NotificationsContext notificationsContext 3: ExploreContext exploreContext } (persisted='true', hasPersonalData='false') ================================================ FILE: cr-mixer/thrift/src/main/thrift/related_tweet.thrift ================================================ namespace java com.twitter.cr_mixer.thriftjava #@namespace scala com.twitter.cr_mixer.thriftscala #@namespace strato com.twitter.cr_mixer include "product.thrift" include "com/twitter/product_mixer/core/client_context.thrift" include "com/twitter/simclusters_v2/identifier.thrift" struct RelatedTweetRequest { 1: required identifier.InternalId internalId 2: required product.Product product 3: required client_context.ClientContext clientContext # RUX LogOut will have clientContext.userId = None 4: optional list excludedTweetIds (personalDataType = 'TweetId') } (persisted='true', hasPersonalData='true') struct RelatedTweet { 1: required i64 tweetId (personalDataType = 'TweetId') 2: optional double score 3: optional i64 authorId (personalDataType = 'UserId') } (persisted='true', hasPersonalData='true') struct RelatedTweetResponse { 1: required list tweets } (persisted='true') ================================================ FILE: cr-mixer/thrift/src/main/thrift/related_video_tweet.thrift ================================================ namespace java com.twitter.cr_mixer.thriftjava #@namespace scala com.twitter.cr_mixer.thriftscala #@namespace strato com.twitter.cr_mixer include "product.thrift" include "com/twitter/product_mixer/core/client_context.thrift" include "com/twitter/simclusters_v2/identifier.thrift" struct RelatedVideoTweetRequest { 1: required identifier.InternalId internalId 2: required product.Product product 3: required client_context.ClientContext clientContext # RUX LogOut will have clientContext.userId = None 4: optional list excludedTweetIds (personalDataType = 'TweetId') } (persisted='true', hasPersonalData='true') struct RelatedVideoTweet { 1: required i64 tweetId (personalDataType = 'TweetId') 2: optional double score } (persisted='true', hasPersonalData='true') struct RelatedVideoTweetResponse { 1: required list tweets } (persisted='true') ================================================ FILE: cr-mixer/thrift/src/main/thrift/scribe.thrift ================================================ namespace java com.twitter.cr_mixer.thriftjava #@namespace scala com.twitter.cr_mixer.thriftscala #@namespace strato com.twitter.cr_mixer include "ads.thrift" include "candidate_generation_key.thrift" include "cr_mixer.thrift" include "metric_tags.thrift" include "product.thrift" include "related_tweet.thrift" include "source_type.thrift" include "uteg.thrift" include "com/twitter/ml/api/data.thrift" include "com/twitter/simclusters_v2/identifier.thrift" struct VITTweetCandidatesScribe { 1: required i64 uuid (personalDataType = 'UniversallyUniqueIdentifierUuid') # RequestUUID - unique scribe id for every request that comes in. Same request but different stages of scribe log (FetchCandidate, Filter, etc) share the same uuid 2: required i64 userId (personalDataType = 'UserId') 3: required list candidates 7: required product.Product product 8: required list impressedBuckets } (persisted='true', hasPersonalData = 'true') struct VITTweetCandidateScribe { 1: required i64 tweetId (personalDataType = 'TweetId') 2: required i64 authorId (personalDataType = 'UserId') 3: required double score 4: required list metricTags } (persisted='true', hasPersonalData = 'true') struct GetTweetsRecommendationsScribe { 1: required i64 uuid (personalDataType = 'UniversallyUniqueIdentifierUuid') # RequestUUID - unique scribe id for every request that comes in. Same request but different stages of scribe log (FetchCandidate, Filter, etc) share the same uuid 2: required i64 userId (personalDataType = 'UserId') 3: required Result result 4: optional i64 traceId 5: optional PerformanceMetrics performanceMetrics 6: optional list impressedBuckets } (persisted='true', hasPersonalData = 'true') struct SourceSignal { # optional, since that the next step covers all info here 1: optional identifier.InternalId id } (persisted='true') struct PerformanceMetrics { 1: optional i64 latencyMs } (persisted='true') struct TweetCandidateWithMetadata { 1: required i64 tweetId (personalDataType = 'TweetId') 2: optional candidate_generation_key.CandidateGenerationKey candidateGenerationKey 3: optional i64 authorId (personalDataType = 'UserId') # only for InterleaveResult for hydrating training data 4: optional double score # score with respect to candidateGenerationKey 5: optional data.DataRecord dataRecord # attach any features to this candidate 6: optional i32 numCandidateGenerationKeys # num CandidateGenerationKeys generating this tweetId } (persisted='true') struct FetchSignalSourcesResult { 1: optional set signals } (persisted='true') struct FetchCandidatesResult { 1: optional list tweets } (persisted='true') struct PreRankFilterResult { 1: optional list tweets } (persisted='true') struct InterleaveResult { 1: optional list tweets } (persisted='true') struct RankResult { 1: optional list tweets } (persisted='true') struct TopLevelApiResult { 1: required i64 timestamp (personalDataType = 'PrivateTimestamp') 2: required cr_mixer.CrMixerTweetRequest request 3: required cr_mixer.CrMixerTweetResponse response } (persisted='true') union Result { 1: FetchSignalSourcesResult fetchSignalSourcesResult 2: FetchCandidatesResult fetchCandidatesResult 3: PreRankFilterResult preRankFilterResult 4: InterleaveResult interleaveResult 5: RankResult rankResult 6: TopLevelApiResult topLevelApiResult } (persisted='true', hasPersonalData = 'true') struct ImpressesedBucketInfo { 1: required i64 experimentId (personalDataType = 'ExperimentId') 2: required string bucketName 3: required i32 version } (persisted='true') ############# RelatedTweets Scribe ############# struct GetRelatedTweetsScribe { 1: required i64 uuid (personalDataType = 'UniversallyUniqueIdentifierUuid') # RequestUUID - unique scribe id for every request that comes in. Same request but different stages of scribe log (FetchCandidate, Filter, etc) share the same uuid 2: required identifier.InternalId internalId 3: required RelatedTweetResult relatedTweetResult 4: optional i64 requesterId (personalDataType = 'UserId') 5: optional i64 guestId (personalDataType = 'GuestId') 6: optional i64 traceId 7: optional PerformanceMetrics performanceMetrics 8: optional list impressedBuckets } (persisted='true', hasPersonalData = 'true') struct RelatedTweetTopLevelApiResult { 1: required i64 timestamp (personalDataType = 'PrivateTimestamp') 2: required related_tweet.RelatedTweetRequest request 3: required related_tweet.RelatedTweetResponse response } (persisted='true') union RelatedTweetResult { 1: RelatedTweetTopLevelApiResult relatedTweetTopLevelApiResult 2: FetchCandidatesResult fetchCandidatesResult 3: PreRankFilterResult preRankFilterResult # results after seqential filters # if later we need rankResult, we can add it here } (persisted='true', hasPersonalData = 'true') ############# UtegTweets Scribe ############# struct GetUtegTweetsScribe { 1: required i64 uuid (personalDataType = 'UniversallyUniqueIdentifierUuid') # RequestUUID - unique scribe id for every request that comes in. Same request but different stages of scribe log (FetchCandidate, Filter, etc) share the same uuid 2: required i64 userId (personalDataType = 'UserId') 3: required UtegTweetResult utegTweetResult 4: optional i64 traceId 5: optional PerformanceMetrics performanceMetrics 6: optional list impressedBuckets } (persisted='true', hasPersonalData = 'true') struct UtegTweetTopLevelApiResult { 1: required i64 timestamp (personalDataType = 'PrivateTimestamp') 2: required uteg.UtegTweetRequest request 3: required uteg.UtegTweetResponse response } (persisted='true') union UtegTweetResult { 1: UtegTweetTopLevelApiResult utegTweetTopLevelApiResult 2: FetchCandidatesResult fetchCandidatesResult # if later we need rankResult, we can add it here } (persisted='true', hasPersonalData = 'true') ############# getAdsRecommendations() Scribe ############# struct GetAdsRecommendationsScribe { 1: required i64 uuid (personalDataType = 'UniversallyUniqueIdentifierUuid') # RequestUUID - unique scribe id for every request that comes in. Same request but different stages of scribe log (FetchCandidate, Filter, etc) share the same uuid 2: required i64 userId (personalDataType = 'UserId') 3: required AdsRecommendationsResult result 4: optional i64 traceId 5: optional PerformanceMetrics performanceMetrics 6: optional list impressedBuckets } (persisted='true', hasPersonalData = 'true') struct AdsRecommendationTopLevelApiResult { 1: required i64 timestamp (personalDataType = 'PrivateTimestamp') 2: required ads.AdsRequest request 3: required ads.AdsResponse response } (persisted='true') union AdsRecommendationsResult{ 1: AdsRecommendationTopLevelApiResult adsRecommendationTopLevelApiResult 2: FetchCandidatesResult fetchCandidatesResult }(persisted='true', hasPersonalData = 'true') ================================================ FILE: cr-mixer/thrift/src/main/thrift/source_type.thrift ================================================ namespace java com.twitter.cr_mixer.thriftjava #@namespace scala com.twitter.cr_mixer.thriftscala #@namespace strato com.twitter.cr_mixer // Due to legacy reason, SourceType used to represent both SourceSignalType and SimilarityEngineType // Hence, you can see several SourceType such as UserInterestedIn, HashSpace, etc. // Moving forward, SourceType will be used for SourceSignalType ONLY. eg., TweetFavorite, UserFollow // We will create a new SimilarityEngineType to separate them. eg., SimClustersANN enum SourceType { // Tweet based Source Signal TweetFavorite = 0 Retweet = 1 TrafficAttribution = 2 // Traffic Attribution will be migrated over in Q3 OriginalTweet = 3 Reply = 4 TweetShare = 5 GoodTweetClick = 6 // total dwell time > N seconds after click on the tweet VideoTweetQualityView = 7 VideoTweetPlayback50 = 8 // UserId based Source Signal (includes both Producer/Consumer) UserFollow = 101 UserRepeatedProfileVisit = 102 CurrentUser_DEPRECATED = 103 RealGraphOon = 104 FollowRecommendation = 105 TwiceUserId = 106 UserTrafficAttributionProfileVisit = 107 GoodProfileClick = 108 // total dwell time > N seconds after click into the profile page // (Notification) Tweet based Source Signal NotificationClick = 201 // (Home) Tweet based Source Signal HomeTweetClick = 301 HomeVideoView = 302 HomeSongbirdShowMore = 303 // Topic based Source Signal TopicFollow = 401 // Deprecated PopularTopic = 402 // Deprecated // Old CR code UserInterestedIn = 501 // Deprecated TwiceInterestedIn = 502 // Deprecated MBCG = 503 // Deprecated HashSpace = 504 // Deprecated // Old CR code Cluster = 601 // Deprecated // Search based Source Signal SearchProfileClick = 701 // Deprecated SearchTweetClick = 702 // Deprecated // Graph based Source StrongTiePrediction = 801 // STP TwiceClustersMembers = 802 Lookalike = 803 // Deprecated RealGraphIn = 804 // Current requester User Id. It is only used for scribing. Placeholder value RequestUserId = 1001 // Current request Tweet Id used in RelatedTweet. Placeholder value RequestTweetId = 1002 // Negative Signals TweetReport = 1101 TweetDontLike = 1102 TweetSeeFewer = 1103 AccountBlock = 1104 AccountMute = 1105 // Aggregated Signals TweetAggregation = 1201 ProducerAggregation = 1202 } (persisted='true', hasPersonalData='true') enum SimilarityEngineType { SimClustersANN = 1 TweetBasedUserTweetGraph = 2 TweetBasedTwHINANN = 3 Follow2VecANN = 4 // ConsumerEmbeddingBasedFollow2Vec QIG = 5 OfflineSimClustersANN = 6 LookalikeUTG_DEPRECATED = 7 ProducerBasedUserTweetGraph = 8 FrsUTG_DEPRECATED = 9 RealGraphOonUTG_DEPRECATED = 10 ConsumerEmbeddingBasedTwHINANN = 11 TwhinCollabFilter = 12 TwiceUTG_DEPRECATED = 13 ConsumerEmbeddingBasedTwoTowerANN = 14 TweetBasedBeTANN = 15 StpUTG_DEPRECATED = 16 UTEG = 17 ROMR = 18 ConsumersBasedUserTweetGraph = 19 TweetBasedUserVideoGraph = 20 CertoTopicTweet = 24 ConsumersBasedUserAdGraph = 25 TweetBasedUserAdGraph = 26 SkitTfgTopicTweet = 27 ConsumerBasedWalsANN = 28 ProducerBasedUserAdGraph = 29 SkitHighPrecisionTopicTweet = 30 SkitInterestBrowserTopicTweet = 31 SkitProducerBasedTopicTweet = 32 ExploreTripOfflineSimClustersTweets = 33 DiffusionBasedTweet = 34 ConsumersBasedUserVideoGraph = 35 // In network EarlybirdRecencyBasedSimilarityEngine = 21 EarlybirdModelBasedSimilarityEngine = 22 EarlybirdTensorflowBasedSimilarityEngine = 23 // Composite TweetBasedUnifiedSimilarityEngine = 1001 ProducerBasedUnifiedSimilarityEngine = 1002 } (persisted='true') ================================================ FILE: cr-mixer/thrift/src/main/thrift/topic_tweet.thrift ================================================ namespace java com.twitter.cr_mixer.thriftjava #@namespace scala com.twitter.cr_mixer.thriftscala #@namespace strato com.twitter.cr_mixer include "com/twitter/product_mixer/core/client_context.thrift" include "product.thrift" include "product_context.thrift" include "source_type.thrift" struct TopicTweetRequest { 1: required client_context.ClientContext clientContext 2: required product.Product product 3: required list topicIds 5: optional product_context.ProductContext productContext 6: optional list excludedTweetIds (personalDataType = 'TweetId') } (persisted='true', hasPersonalData='true') struct TopicTweet { 1: required i64 tweetId (personalDataType = 'TweetId') 2: required double score 3: required source_type.SimilarityEngineType similarityEngineType } (persisted='true', hasPersonalData = 'true') struct TopicTweetResponse { 1: required map> tweets } (persisted='true') ================================================ FILE: cr-mixer/thrift/src/main/thrift/uteg.thrift ================================================ namespace java com.twitter.cr_mixer.thriftjava #@namespace scala com.twitter.cr_mixer.thriftscala #@namespace strato com.twitter.cr_mixer include "product.thrift" include "product_context.thrift" include "com/twitter/product_mixer/core/client_context.thrift" include "com/twitter/recos/recos_common.thrift" struct UtegTweetRequest { 1: required client_context.ClientContext clientContext 2: required product.Product product # Product-specific parameters should be placed in the Product Context 3: optional product_context.ProductContext productContext 4: optional list excludedTweetIds (personalDataType = 'TweetId') } (persisted='true', hasPersonalData='true') struct UtegTweet { // tweet id 1: required i64 tweetId(personalDataType = 'TweetId') // sum of weights of seed users who engaged with the tweet. // If a user engaged with the same tweet twice, liked it and retweeted it, then his/her weight was counted twice. 2: required double score // user social proofs per engagement type 3: required map> socialProofByType(personalDataTypeKey='EngagementTypePrivate', personalDataTypeValue='UserId') } (persisted='true', hasPersonalData = 'true') struct UtegTweetResponse { 1: required list tweets } (persisted='true') ================================================ FILE: cr-mixer/thrift/src/main/thrift/validation.thrift ================================================ namespace java com.twitter.cr_mixer.thriftjava #@namespace scala com.twitter.cr_mixer.thriftscala #@namespace strato com.twitter.cr_mixer // ValidationErrorCode is used to identify classes of client errors returned from a Product Mixer // service. Use [[PipelineFailureExceptionMapper]] to adapt pipeline failures into thrift errors. enum ValidationErrorCode { PRODUCT_DISABLED = 1 PLACEHOLDER_2 = 2 } (hasPersonalData='false') exception ValidationException { 1: ValidationErrorCode errorCode 2: string msg } (hasPersonalData='false') exception ValidationExceptionList { 1: list errors } (hasPersonalData='false') ================================================ FILE: follow-recommendations-service/BUILD ================================================ # Without this alias, library :follow-recommendations-service_lib would conflict with :bin alias( name = "follow-recommendations-service", target = ":follow-recommendations-service_lib", ) target( name = "follow-recommendations-service_lib", dependencies = [ "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", ], ) jvm_binary( name = "bin", basename = "follow-recommendations-service", main = "com.twitter.follow_recommendations.FollowRecommendationsServiceThriftServerMain", runtime_platform = "java11", tags = ["bazel-compatible"], dependencies = [ ":follow-recommendations-service", "3rdparty/jvm/ch/qos/logback:logback-classic", "finagle/finagle-zipkin-scribe/src/main/scala", "finatra/inject/inject-logback/src/main/scala", "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", "twitter-server-internal/src/main/scala", "twitter-server/logback-classic/src/main/scala", ], ) # Aurora Workflows build phase convention requires a jvm_app named with ${project-name}-app jvm_app( name = "follow-recommendations-service-app", archive = "zip", binary = ":bin", bundles = [ bundle( fileset = [ "server/src/main/resources/*", "server/src/main/resources/**/*", ], owning_target = "follow-recommendations-service/server/src/main/resources:frs_resources", relative_to = "server/src/main/resources", ), ], tags = ["bazel-compatible"], ) ================================================ FILE: follow-recommendations-service/CONFIG.ini ================================================ [code-coverage] package = com.twitter.follow_recommendations [docbird] project_name = follow-recommendations-service project_type = service ; example settings: ; ; project_name = fluffybird ; description = fluffybird is a service for fluffing up feathers. ; tags = python,documentation,fluffybird ; project_type = service ; - allowed options: essay, library, service, hub, cookbook, styleguide, policy ; owner_links = roster ; - allowed options: roster, find, email ; scrolling_tocs = yes ; comments = yes ; verifications = yes ; support_widget = yes ; health_score = yes ; sticky_sidebar = no [jira] project = CJREL ================================================ FILE: follow-recommendations-service/README.md ================================================ # Follow Recommendations Service ## Introduction to the Follow Recommendations Service (FRS) The Follow Recommendations Service (FRS) is a robust recommendation engine designed to provide users with personalized suggestions for accounts to follow. At present, FRS supports Who-To-Follow (WTF) module recommendations across a variety of Twitter product interfaces. Additionally, by suggesting tweet authors, FRS also delivers FutureGraph tweet recommendations, which consist of tweets from accounts that users may be interested in following in the future. ## Design The system is tailored to accommodate diverse use cases, such as Post New-User-Experience (NUX), advertisements, FutureGraph tweets, and more. Each use case features a unique display location identifier. To view all display locations, refer to the following path: `follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala`. Recommendation steps are customized according to each display location. Common and high-level steps are encapsulated within the "RecommendationFlow," which includes operations like candidate generation, ranker selection, filtering, transformation, and beyond. To explore all flows, refer to this path: `follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows`. For each product (corresponding to a display location), one or multiple flows can be selected to generate candidates based on code and configurations. To view all products, refer to the following path: `follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs`. The FRS overview diagram is depicted below: ![FRS_architecture.png](FRS_architecture.png) ### Candidate Generation During this step, FRS utilizes various user signals and algorithms to identify candidates from all Twitter accounts. The candidate source folder is located at `follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/`, with a README file provided within each candidate source folder. ### Filtering In this phase, FRS applies different filtering logic after generating account candidates to improve quality and health. Filtering may occur before and/or after the ranking step, with heavier filtering logic (e.g., higher latency) typically applied after the ranking step. The filters' folder is located at `follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates`. ### Ranking During this step, FRS employs both Machine Learning (ML) and heuristic rule-based candidate ranking. For the ML ranker, ML features are fetched beforehand (i.e., feature hydration), and a DataRecord (the Twitter-standard Machine Learning data format used to represent feature data, labels, and predictions when training or serving) is constructed for each pair. These pairs are then sent to a separate ML prediction service, which houses the ML model trained offline. The ML prediction service returns a prediction score, representing the probability that a user will follow and engage with the candidate. This score is a weighted sum of p(follow|recommendation) and p(positive engagement|follow), and FRS uses this score to rank the candidates. The rankers' folder is located at `follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers`. ### Transform In this phase, the sequence of candidates undergoes necessary transformations, such as deduplication, attaching social proof (i.e., "followed by XX user"), adding tracking tokens, and more. The transformers' folder can be found at `follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms`. ### Truncation During this final step, FRS trims the candidate pool to a specified size. This process ensures that only the most relevant and engaging candidates are presented to users while maintaining an optimal user experience. By implementing these comprehensive steps and adapting to various use cases, the Follow Recommendations Service (FRS) effectively curates tailored suggestions for Twitter users, enhancing their overall experience and promoting meaningful connections within the platform. ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/guava", "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", "finagle/finagle-core/src/main", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "hermit/hermit-core/src/main/scala/com/twitter/hermit/model", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation", "stitch/stitch-core", ], exports = [ "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/CandidateSourceRegistry.scala ================================================ package com.twitter.follow_recommendations.common.base import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource.toEnriched import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier // a helper structure to register and select candidate sources based on identifiers trait CandidateSourceRegistry[Target, Candidate] { val statsReceiver: StatsReceiver def sources: Set[CandidateSource[Target, Candidate]] final lazy val candidateSources: Map[ CandidateSourceIdentifier, CandidateSource[Target, Candidate] ] = { val map = sources.map { c => c.identifier -> c.observe(statsReceiver) }.toMap if (map.size != sources.size) { throw new IllegalArgumentException("Duplicate Candidate Source Identifiers") } map } def select( identifiers: Set[CandidateSourceIdentifier] ): Set[CandidateSource[Target, Candidate]] = { // fails loud if the candidate source is not registered identifiers.map(candidateSources(_)) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/EnrichedCandidateSource.scala ================================================ package com.twitter.follow_recommendations.common.base import com.twitter.finagle.stats.StatsReceiver import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.stitch.Stitch import com.twitter.util.Duration import com.twitter.util.TimeoutException import scala.language.implicitConversions class EnrichedCandidateSource[Target, Candidate](original: CandidateSource[Target, Candidate]) { /** * Gate the candidate source based on the Predicate of target. * It returns results only if the predicate returns Valid. * * @param predicate * @return */ def gate(predicate: Predicate[Target]): CandidateSource[Target, Candidate] = { throw new UnsupportedOperationException() } def observe(statsReceiver: StatsReceiver): CandidateSource[Target, Candidate] = { val originalIdentifier = original.identifier val stats = statsReceiver.scope(originalIdentifier.name) new CandidateSource[Target, Candidate] { val identifier = originalIdentifier override def apply(target: Target): Stitch[Seq[Candidate]] = { StatsUtil.profileStitchSeqResults[Candidate](original(target), stats) } } } /** * Map target type into new target type (1 to optional mapping) */ def stitchMapKey[Target2]( targetMapper: Target2 => Stitch[Option[Target]] ): CandidateSource[Target2, Candidate] = { val targetsMapper: Target2 => Stitch[Seq[Target]] = { target => targetMapper(target).map(_.toSeq) } stitchMapKeys(targetsMapper) } /** * Map target type into new target type (1 to many mapping) */ def stitchMapKeys[Target2]( targetMapper: Target2 => Stitch[Seq[Target]] ): CandidateSource[Target2, Candidate] = { new CandidateSource[Target2, Candidate] { val identifier = original.identifier override def apply(target: Target2): Stitch[Seq[Candidate]] = { for { mappedTargets <- targetMapper(target) results <- Stitch.traverse(mappedTargets)(original(_)) } yield results.flatten } } } /** * Map target type into new target type (1 to many mapping) */ def mapKeys[Target2]( targetMapper: Target2 => Seq[Target] ): CandidateSource[Target2, Candidate] = { val stitchMapper: Target2 => Stitch[Seq[Target]] = { target => Stitch.value(targetMapper(target)) } stitchMapKeys(stitchMapper) } /** * Map candidate types to new type based on candidateMapper */ def mapValues[Candidate2]( candidateMapper: Candidate => Stitch[Option[Candidate2]] ): CandidateSource[Target, Candidate2] = { new CandidateSource[Target, Candidate2] { val identifier = original.identifier override def apply(target: Target): Stitch[Seq[Candidate2]] = { original(target).flatMap { candidates => val results = Stitch.traverse(candidates)(candidateMapper(_)) results.map(_.flatten) } } } } /** * Map candidate types to new type based on candidateMapper */ def mapValue[Candidate2]( candidateMapper: Candidate => Candidate2 ): CandidateSource[Target, Candidate2] = { val stitchMapper: Candidate => Stitch[Option[Candidate2]] = { c => Stitch.value(Some(candidateMapper(c))) } mapValues(stitchMapper) } /** * This method wraps the candidate source in a designated timeout so that a single candidate * source does not result in a timeout for the entire flow */ def within( candidateTimeout: Duration, statsReceiver: StatsReceiver ): CandidateSource[Target, Candidate] = { val originalIdentifier = original.identifier val timeoutCounter = statsReceiver.counter(originalIdentifier.name, "timeout") new CandidateSource[Target, Candidate] { val identifier = originalIdentifier override def apply(target: Target): Stitch[Seq[Candidate]] = { original .apply(target) .within(candidateTimeout)(com.twitter.finagle.util.DefaultTimer) .rescue { case _: TimeoutException => timeoutCounter.incr() Stitch.Nil } } } } def failOpenWithin( candidateTimeout: Duration, statsReceiver: StatsReceiver ): CandidateSource[Target, Candidate] = { val originalIdentifier = original.identifier val timeoutCounter = statsReceiver.counter(originalIdentifier.name, "timeout") new CandidateSource[Target, Candidate] { val identifier = originalIdentifier override def apply(target: Target): Stitch[Seq[Candidate]] = { original .apply(target) .within(candidateTimeout)(com.twitter.finagle.util.DefaultTimer) .handle { case _: TimeoutException => timeoutCounter.incr() Seq.empty case e: Exception => statsReceiver .scope("candidate_source_error").scope(originalIdentifier.name).counter( e.getClass.getSimpleName).incr Seq.empty } } } } } object EnrichedCandidateSource { implicit def toEnriched[K, V](original: CandidateSource[K, V]): EnrichedCandidateSource[K, V] = new EnrichedCandidateSource(original) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/ParamPredicate.scala ================================================ package com.twitter.follow_recommendations.common.base import com.twitter.follow_recommendations.common.models.FilterReason.ParamReason import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.timelines.configapi.Param case class ParamPredicate[Request <: HasParams](param: Param[Boolean]) extends Predicate[Request] { def apply(request: Request): Stitch[PredicateResult] = { if (request.params(param)) { Stitch.value(PredicateResult.Valid) } else { Stitch.value(PredicateResult.Invalid(Set(ParamReason(param.statName)))) } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/Predicate.scala ================================================ package com.twitter.follow_recommendations.common.base import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.models.FilterReason import com.twitter.stitch.Arrow import com.twitter.stitch.Stitch trait Predicate[-Q] { def apply(item: Q): Stitch[PredicateResult] def arrow: Arrow[Q, PredicateResult] = Arrow.apply(apply) def map[K](mapper: K => Q): Predicate[K] = Predicate(arrow.contramap(mapper)) /** * check the predicate results for a batch of items for convenience. * * mark it as final to avoid potential abuse usage */ final def batch(items: Seq[Q]): Stitch[Seq[PredicateResult]] = { this.arrow.traverse(items) } /** * Syntax sugar for functions which take in 2 inputs as a tuple. */ def apply[Q1, Q2](item1: Q1, item2: Q2)(implicit ev: ((Q1, Q2)) => Q): Stitch[PredicateResult] = { apply((item1, item2)) } /** * Runs the predicates in sequence. The returned predicate will return true iff both the predicates return true. * ie. it is an AND operation * * We short-circuit the evaluation, ie we don't evaluate the 2nd predicate if the 1st is false * * @param p predicate to run in sequence * * @return a new predicate object that represents the logical AND of both predicates */ def andThen[Q1 <: Q](p: Predicate[Q1]): Predicate[Q1] = { Predicate({ query: Q1 => apply(query).flatMap { case PredicateResult.Valid => p(query) case PredicateResult.Invalid(reasons) => Stitch.value(PredicateResult.Invalid(reasons)) } }) } /** * Creates a predicate which runs the current & given predicate in sequence. * The returned predicate will return true if either current or given predicate returns true. * That is, given predicate will be only run if current predicate returns false. * * @param p predicate to run in sequence * * @return new predicate object that represents the logical OR of both predicates. * if both are invalid, the reason would be the set of all invalid reasons. */ def or[Q1 <: Q](p: Predicate[Q1]): Predicate[Q1] = { Predicate({ query: Q1 => apply(query).flatMap { case PredicateResult.Valid => Stitch.value(PredicateResult.Valid) case PredicateResult.Invalid(reasons) => p(query).flatMap { case PredicateResult.Valid => Stitch.value(PredicateResult.Valid) case PredicateResult.Invalid(newReasons) => Stitch.value(PredicateResult.Invalid(reasons ++ newReasons)) } } }) } /* * Runs the predicate only if the provided predicate is valid, otherwise returns valid. * */ def gate[Q1 <: Q](gatingPredicate: Predicate[Q1]): Predicate[Q1] = { Predicate { query: Q1 => gatingPredicate(query).flatMap { result => if (result == PredicateResult.Valid) { apply(query) } else { Stitch.value(PredicateResult.Valid) } } } } def observe(statsReceiver: StatsReceiver): Predicate[Q] = Predicate( StatsUtil.profilePredicateResult(this.arrow, statsReceiver)) def convertToFailOpenWithResultType(resultType: PredicateResult): Predicate[Q] = { Predicate { query: Q => apply(query).handle { case _: Exception => resultType } } } } class TruePredicate[Q] extends Predicate[Q] { override def apply(item: Q): Stitch[PredicateResult] = Predicate.AlwaysTrueStitch } class FalsePredicate[Q](reason: FilterReason) extends Predicate[Q] { val InvalidResult = Stitch.value(PredicateResult.Invalid(Set(reason))) override def apply(item: Q): Stitch[PredicateResult] = InvalidResult } object Predicate { val AlwaysTrueStitch = Stitch.value(PredicateResult.Valid) val NumBatchesStat = "num_batches_stats" val NumBatchesCount = "num_batches" def apply[Q](func: Q => Stitch[PredicateResult]): Predicate[Q] = new Predicate[Q] { override def apply(item: Q): Stitch[PredicateResult] = func(item) override val arrow: Arrow[Q, PredicateResult] = Arrow(func) } def apply[Q](outerArrow: Arrow[Q, PredicateResult]): Predicate[Q] = new Predicate[Q] { override def apply(item: Q): Stitch[PredicateResult] = arrow(item) override val arrow: Arrow[Q, PredicateResult] = outerArrow } /** * Given some items, this function * 1. chunks them up in groups * 2. lazily applies a predicate on each group * 3. filters based on the predicate * 4. takes first numToTake items. * * If numToTake is satisfied, then any later predicates are not called. * * @param items items of type Q * @param predicate predicate that determines whether an item is acceptable * @param batchSize batch size to call the predicate with * @param numToTake max number of items to return * @param stats stats receiver * @tparam Q type of item * * @return a future of K items */ def batchFilterTake[Q]( items: Seq[Q], predicate: Predicate[Q], batchSize: Int, numToTake: Int, stats: StatsReceiver ): Stitch[Seq[Q]] = { def take( input: Iterator[Stitch[Seq[Q]]], prev: Seq[Q], takeSize: Int, numOfBatch: Int ): Stitch[(Seq[Q], Int)] = { if (input.hasNext) { val currFut = input.next() currFut.flatMap { curr => val taken = curr.take(takeSize) val combined = prev ++ taken if (taken.size < takeSize) take(input, combined, takeSize - taken.size, numOfBatch + 1) else Stitch.value((combined, numOfBatch + 1)) } } else { Stitch.value((prev, numOfBatch)) } } val batchedItems = items.view.grouped(batchSize) val batchedFutures = batchedItems.map { batch => Stitch.traverse(batch)(predicate.apply).map { conds => (batch.zip(conds)).withFilter(_._2.value).map(_._1) } } take(batchedFutures, Nil, numToTake, 0).map { case (filtered: Seq[Q], numOfBatch: Int) => stats.stat(NumBatchesStat).add(numOfBatch) stats.counter(NumBatchesCount).incr(numOfBatch) filtered } } /** * filter a list of items based on the predicate * * @param items a list of items * @param predicate predicate of the item * @tparam Q item type * @return the list of items that satisfy the predicate */ def filter[Q](items: Seq[Q], predicate: Predicate[Q]): Stitch[Seq[Q]] = { predicate.batch(items).map { results => items.zip(results).collect { case (item, PredicateResult.Valid) => item } } } /** * filter a list of items based on the predicate given the target * * @param target target item * @param items a list of items * @param predicate predicate of the (target, item) pair * @tparam Q item type * @return the list of items that satisfy the predicate given the target */ def filter[T, Q](target: T, items: Seq[Q], predicate: Predicate[(T, Q)]): Stitch[Seq[Q]] = { predicate.batch(items.map(i => (target, i))).map { results => items.zip(results).collect { case (item, PredicateResult.Valid) => item } } } /** * Returns a predicate, where an element is true iff it that element is true for all input predicates. * ie. it is an AND operation * * This is done concurrently. * * @param predicates list of predicates * @tparam Q Type parameter * * @return new predicate object that is the logical "and" of the input predicates */ def andConcurrently[Q](predicates: Seq[Predicate[Q]]): Predicate[Q] = { Predicate { query: Q => Stitch.traverse(predicates)(p => p(query)).map { predicateResults => val allInvalid = predicateResults .collect { case PredicateResult.Invalid(reason) => reason } if (allInvalid.isEmpty) { PredicateResult.Valid } else { val allInvalidReasons = allInvalid.reduce(_ ++ _) PredicateResult.Invalid(allInvalidReasons) } } } } } /** * applies the underlying predicate when the param is on. */ abstract class GatedPredicateBase[Q]( underlyingPredicate: Predicate[Q], stats: StatsReceiver = NullStatsReceiver) extends Predicate[Q] { def gate(item: Q): Boolean val underlyingPredicateTotal = stats.counter("underlying_total") val underlyingPredicateValid = stats.counter("underlying_valid") val underlyingPredicateInvalid = stats.counter("underlying_invalid") val notGatedCounter = stats.counter("not_gated") val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) override def apply(item: Q): Stitch[PredicateResult] = { if (gate(item)) { underlyingPredicateTotal.incr() underlyingPredicate(item) } else { notGatedCounter.incr() ValidStitch } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/PredicateResult.scala ================================================ package com.twitter.follow_recommendations.common.base import com.twitter.follow_recommendations.common.models.FilterReason sealed trait PredicateResult { def value: Boolean } object PredicateResult { case object Valid extends PredicateResult { override val value = true } case class Invalid(reasons: Set[FilterReason] = Set.empty[FilterReason]) extends PredicateResult { override val value = false } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/Ranker.scala ================================================ package com.twitter.follow_recommendations.common.base import com.twitter.finagle.stats.StatsReceiver import com.twitter.stitch.Stitch import com.twitter.util.Duration import com.twitter.util.TimeoutException /** * Ranker is a special kind of transform that would only change the order of a list of items. * If a single item is given, it "may" attach additional scoring information to the item. * * @tparam Target target to recommend the candidates * @tparam Candidate candidate type to rank */ trait Ranker[Target, Candidate] extends Transform[Target, Candidate] { ranker => def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] override def transform(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] = { rank(target, candidates) } override def observe(statsReceiver: StatsReceiver): Ranker[Target, Candidate] = { val originalRanker = this new Ranker[Target, Candidate] { override def rank(target: Target, items: Seq[Candidate]): Stitch[Seq[Candidate]] = { statsReceiver.counter(Transform.InputCandidatesCount).incr(items.size) statsReceiver.stat(Transform.InputCandidatesStat).add(items.size) StatsUtil.profileStitchSeqResults(originalRanker.rank(target, items), statsReceiver) } } } def reverse: Ranker[Target, Candidate] = new Ranker[Target, Candidate] { def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] = ranker.rank(target, candidates).map(_.reverse) } def andThen(other: Ranker[Target, Candidate]): Ranker[Target, Candidate] = { val original = this new Ranker[Target, Candidate] { def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] = { original.rank(target, candidates).flatMap { results => other.rank(target, results) } } } } /** * This method wraps the Ranker in a designated timeout. * If the ranker timeouts, it would return the original candidates directly, * instead of failing the whole recommendation flow */ def within(timeout: Duration, statsReceiver: StatsReceiver): Ranker[Target, Candidate] = { val timeoutCounter = statsReceiver.counter("timeout") val original = this new Ranker[Target, Candidate] { override def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] = { original .rank(target, candidates) .within(timeout)(com.twitter.finagle.util.DefaultTimer) .rescue { case _: TimeoutException => timeoutCounter.incr() Stitch.value(candidates) } } } } } object Ranker { def chain[Target, Candidate]( transformer: Transform[Target, Candidate], ranker: Ranker[Target, Candidate] ): Ranker[Target, Candidate] = { new Ranker[Target, Candidate] { def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] = { transformer .transform(target, candidates) .flatMap { results => ranker.rank(target, results) } } } } } class IdentityRanker[Target, Candidate] extends Ranker[Target, Candidate] { def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] = Stitch.value(candidates) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/RecommendationFlow.scala ================================================ package com.twitter.follow_recommendations.common.base import com.twitter.finagle.stats.StatsReceiver import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineResult import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver import com.twitter.stitch.Stitch /** * configs for results generated from the recommendation flow * * @param desiredCandidateCount num of desired candidates to return * @param batchForCandidatesCheck batch size for candidates check */ case class RecommendationResultsConfig(desiredCandidateCount: Int, batchForCandidatesCheck: Int) trait BaseRecommendationFlow[Target, Candidate <: UniversalNoun[Long]] { val identifier = RecommendationPipelineIdentifier("RecommendationFlow") def process( pipelineRequest: Target ): Stitch[RecommendationPipelineResult[Candidate, Seq[Candidate]]] def mapKey[Target2](fn: Target2 => Target): BaseRecommendationFlow[Target2, Candidate] = { val original = this new BaseRecommendationFlow[Target2, Candidate] { override def process( pipelineRequest: Target2 ): Stitch[RecommendationPipelineResult[Candidate, Seq[Candidate]]] = original.process(fn(pipelineRequest)) } } } /** * Defines a typical recommendation flow to fetch, filter, rank and transform candidates. * * 1. targetEligibility: determine the eligibility of target request * 2. candidateSources: fetch candidates from candidate sources based on target type * 3. preRankerCandidateFilter: light filtering of candidates * 4. ranker: ranking of candidates (could be composed of multiple stages, light ranking, heavy ranking and etc) * 5. postRankerTransform: deduping, grouping, rule based promotion / demotions and etc * 6. validateCandidates: heavy filters to determine the eligibility of the candidates. * will only be applied to candidates that we expect to return. * 7. transformResults: transform the individual candidates into desired format (e.g. hydrate social proof) * * Note that the actual implementations may not need to implement all the steps if not needed * (could just leave to IdentityRanker if ranking is not needed). * * Theoretically, the actual implementation could override the above flow to add * more steps (e.g. add a transform step before ranking). * But it is recommended to add the additional steps into this base flow if the step proves * to have significant justification, or merge it into an existing step if it is a minor change. * * @tparam Target type of target request * @tparam Candidate type of candidate to return */ trait RecommendationFlow[Target, Candidate <: UniversalNoun[Long]] extends BaseRecommendationFlow[Target, Candidate] with SideEffectsUtil[Target, Candidate] { /** * optionally update or enrich the request before executing the flows */ protected def updateTarget(target: Target): Stitch[Target] = Stitch.value(target) /** * check if the target is eligible for the flow */ protected def targetEligibility: Predicate[Target] /** * define the candidate sources that should be used for the given target */ protected def candidateSources(target: Target): Seq[CandidateSource[Target, Candidate]] /** * filter invalid candidates before the ranking phase. */ protected def preRankerCandidateFilter: Predicate[(Target, Candidate)] /** * rank the candidates */ protected def selectRanker(target: Target): Ranker[Target, Candidate] /** * transform the candidates after ranking (e.g. dedupping, grouping and etc) */ protected def postRankerTransform: Transform[Target, Candidate] /** * filter invalid candidates before returning the results. * * Some heavy filters e.g. SGS filter could be applied in this step */ protected def validateCandidates: Predicate[(Target, Candidate)] /** * transform the candidates into results and return */ protected def transformResults: Transform[Target, Candidate] /** * configuration for recommendation results */ protected def resultsConfig(target: Target): RecommendationResultsConfig /** * track the quality factor the recommendation pipeline */ protected def qualityFactorObserver: Option[QualityFactorObserver] = None def statsReceiver: StatsReceiver /** * high level monitoring for the whole flow * (make sure to add monitoring for each individual component by yourself) * * additional candidates: count, stats, non_empty_count * target eligibility: latency, success, failures, request, count, valid_count, invalid_count, invalid_reasons * candidate generation: latency, success, failures, request, count, non_empty_count, results_stat * pre ranker filter: latency, success, failures, request, count, non_empty_count, results_stat * ranker: latency, success, failures, request, count, non_empty_count, results_stat * post ranker: latency, success, failures, request, count, non_empty_count, results_stat * filter and take: latency, success, failures, request, count, non_empty_count, results_stat, batch count * transform results: latency, success, failures, request, count, non_empty_count, results_stat */ import RecommendationFlow._ lazy val additionalCandidatesStats = statsReceiver.scope(AdditionalCandidatesStats) lazy val targetEligibilityStats = statsReceiver.scope(TargetEligibilityStats) lazy val candidateGenerationStats = statsReceiver.scope(CandidateGenerationStats) lazy val preRankerFilterStats = statsReceiver.scope(PreRankerFilterStats) lazy val rankerStats = statsReceiver.scope(RankerStats) lazy val postRankerTransformStats = statsReceiver.scope(PostRankerTransformStats) lazy val filterAndTakeStats = statsReceiver.scope(FilterAndTakeStats) lazy val transformResultsStats = statsReceiver.scope(TransformResultsStats) lazy val overallStats = statsReceiver.scope(OverallStats) import StatsUtil._ override def process( pipelineRequest: Target ): Stitch[RecommendationPipelineResult[Candidate, Seq[Candidate]]] = { observeStitchQualityFactor( profileStitchSeqResults( updateTarget(pipelineRequest).flatMap { target => profilePredicateResult(targetEligibility(target), targetEligibilityStats).flatMap { case PredicateResult.Valid => processValidTarget(target, Seq.empty) case PredicateResult.Invalid(_) => Stitch.Nil } }, overallStats ).map { candidates => RecommendationPipelineResult.empty.withResult(candidates) }, qualityFactorObserver, overallStats ) } protected def processValidTarget( target: Target, additionalCandidates: Seq[Candidate] ): Stitch[Seq[Candidate]] = { /** * A basic recommendation flow looks like this: * * 1. fetch candidates from candidate sources * 2. blend candidates with existing candidates * 3. filter the candidates (light filters) before ranking * 4. ranking * 5. filter and truncate the candidates using postRankerCandidateFilter * 6. transform the candidates based on product requirement */ val candidateSourcesToFetch = candidateSources(target) for { candidates <- profileStitchSeqResults( Stitch.traverse(candidateSourcesToFetch)(_(target)).map(_.flatten), candidateGenerationStats ) mergedCandidates = profileSeqResults(additionalCandidates, additionalCandidatesStats) ++ candidates filteredCandidates <- profileStitchSeqResults( Predicate.filter(target, mergedCandidates, preRankerCandidateFilter), preRankerFilterStats ) rankedCandidates <- profileStitchSeqResults( selectRanker(target).rank(target, filteredCandidates), rankerStats ) transformed <- profileStitchSeqResults( postRankerTransform.transform(target, rankedCandidates), postRankerTransformStats ) truncated <- profileStitchSeqResults( take(target, transformed, resultsConfig(target)), filterAndTakeStats ) results <- profileStitchSeqResults( transformResults.transform(target, truncated), transformResultsStats ) _ <- applySideEffects( target, candidateSourcesToFetch, candidates, mergedCandidates, filteredCandidates, rankedCandidates, transformed, truncated, results) } yield results } private[this] def take( target: Target, candidates: Seq[Candidate], config: RecommendationResultsConfig ): Stitch[Seq[Candidate]] = { Predicate .batchFilterTake( candidates.map(c => (target, c)), validateCandidates, config.batchForCandidatesCheck, config.desiredCandidateCount, statsReceiver ).map(_.map(_._2)) } } object RecommendationFlow { val AdditionalCandidatesStats = "additional_candidates" val TargetEligibilityStats = "target_eligibility" val CandidateGenerationStats = "candidate_generation" val PreRankerFilterStats = "pre_ranker_filter" val RankerStats = "ranker" val PostRankerTransformStats = "post_ranker_transform" val FilterAndTakeStats = "filter_and_take" val TransformResultsStats = "transform_results" val OverallStats = "overall" } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/SideEffectsUtil.scala ================================================ package com.twitter.follow_recommendations.common.base import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.stitch.Stitch /** * SideEffectsUtil applies side effects to the intermediate candidate results from a recommendation flow pipeline. * * @tparam Target target to recommend the candidates * @tparam Candidate candidate type to rank */ trait SideEffectsUtil[Target, Candidate] { def applySideEffects( target: Target, candidateSources: Seq[CandidateSource[Target, Candidate]], candidatesFromCandidateSources: Seq[Candidate], mergedCandidates: Seq[Candidate], filteredCandidates: Seq[Candidate], rankedCandidates: Seq[Candidate], transformedCandidates: Seq[Candidate], truncatedCandidates: Seq[Candidate], results: Seq[Candidate] ): Stitch[Unit] = Stitch.Unit } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/StatsUtil.scala ================================================ package com.twitter.follow_recommendations.common.base import com.twitter.finagle.stats.Stat import com.twitter.finagle.stats.StatsReceiver import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver import com.twitter.stitch.Arrow import com.twitter.stitch.Stitch import com.twitter.util.Stopwatch import java.util.concurrent.TimeUnit import scala.util.control.NonFatal object StatsUtil { val LatencyName = "latency_ms" val RequestName = "requests" val SuccessName = "success" val FailureName = "failures" val ResultsName = "results" val ResultsStat = "results_stat" val EmptyResultsName = "empty" val NonEmptyResultsName = "non_empty" val ValidCount = "valid" val InvalidCount = "invalid" val InvalidHasReasons = "has_reasons" val Reasons = "reasons" val QualityFactorStat = "quality_factor_stat" val QualityFactorCounts = "quality_factor_counts" /** * Helper function for timing a stitch, returning the original stitch. */ def profileStitch[T](stitch: Stitch[T], stat: StatsReceiver): Stitch[T] = { Stitch .time(stitch) .map { case (response, stitchRunDuration) => stat.counter(RequestName).incr() stat.stat(LatencyName).add(stitchRunDuration.inMilliseconds) response .onSuccess { _ => stat.counter(SuccessName).incr() } .onFailure { e => stat.counter(FailureName).incr() stat.scope(FailureName).counter(getCleanClassName(e)).incr() } } .lowerFromTry } /** * Helper function for timing an arrow, returning the original arrow. */ def profileArrow[T, U](arrow: Arrow[T, U], stat: StatsReceiver): Arrow[T, U] = { Arrow .time(arrow) .map { case (response, stitchRunDuration) => stat.counter(RequestName).incr() stat.stat(LatencyName).add(stitchRunDuration.inMilliseconds) response .onSuccess { _ => stat.counter(SuccessName).incr() } .onFailure { e => stat.counter(FailureName).incr() stat.scope(FailureName).counter(getCleanClassName(e)).incr() } } .lowerFromTry } /** * Helper function to count and track the distribution of results */ def profileResults[T](results: T, stat: StatsReceiver, size: T => Int): T = { val numResults = size(results) stat.counter(ResultsName).incr(numResults) if (numResults == 0) { stat.counter(EmptyResultsName).incr() results } else { stat.stat(ResultsStat).add(numResults) stat.counter(NonEmptyResultsName).incr() results } } /** * Helper function to count and track the distribution of a list of results */ def profileSeqResults[T](results: Seq[T], stat: StatsReceiver): Seq[T] = { profileResults[Seq[T]](results, stat, _.size) } /** * Helper function for timing a stitch and count the number of results, returning the original stitch. */ def profileStitchResults[T](stitch: Stitch[T], stat: StatsReceiver, size: T => Int): Stitch[T] = { profileStitch(stitch, stat).onSuccess { results => profileResults(results, stat, size) } } /** * Helper function for timing an arrow and count the number of results, returning the original arrow. */ def profileArrowResults[T, U]( arrow: Arrow[T, U], stat: StatsReceiver, size: U => Int ): Arrow[T, U] = { profileArrow(arrow, stat).onSuccess { results => profileResults(results, stat, size) } } /** * Helper function for timing a stitch and count a seq of results, returning the original stitch. */ def profileStitchSeqResults[T](stitch: Stitch[Seq[T]], stat: StatsReceiver): Stitch[Seq[T]] = { profileStitchResults[Seq[T]](stitch, stat, _.size) } /** * Helper function for timing a stitch and count optional results, returning the original stitch. */ def profileStitchOptionalResults[T]( stitch: Stitch[Option[T]], stat: StatsReceiver ): Stitch[Option[T]] = { profileStitchResults[Option[T]](stitch, stat, _.size) } /** * Helper function for timing a stitch and count a map of results, returning the original stitch. */ def profileStitchMapResults[K, V]( stitch: Stitch[Map[K, V]], stat: StatsReceiver ): Stitch[Map[K, V]] = { profileStitchResults[Map[K, V]](stitch, stat, _.size) } def getCleanClassName(obj: Object): String = obj.getClass.getSimpleName.stripSuffix("$") /** * Helper function for timing a stitch and count a list of PredicateResult */ def profilePredicateResults( predicateResult: Stitch[Seq[PredicateResult]], statsReceiver: StatsReceiver ): Stitch[Seq[PredicateResult]] = { profileStitch[Seq[PredicateResult]]( predicateResult, statsReceiver ).onSuccess { _.map { case PredicateResult.Valid => statsReceiver.counter(ValidCount).incr() case PredicateResult.Invalid(reasons) => statsReceiver.counter(InvalidCount).incr() reasons.map { filterReason => statsReceiver.counter(InvalidHasReasons).incr() statsReceiver.scope(Reasons).counter(filterReason.reason).incr() } } } } /** * Helper function for timing a stitch and count individual PredicateResult */ def profilePredicateResult( predicateResult: Stitch[PredicateResult], statsReceiver: StatsReceiver ): Stitch[PredicateResult] = { profilePredicateResults( predicateResult.map(Seq(_)), statsReceiver ).map(_.head) } /** * Helper function for timing an arrow and count a list of PredicateResult */ def profilePredicateResults[Q]( predicateResult: Arrow[Q, Seq[PredicateResult]], statsReceiver: StatsReceiver ): Arrow[Q, Seq[PredicateResult]] = { profileArrow[Q, Seq[PredicateResult]]( predicateResult, statsReceiver ).onSuccess { _.map { case PredicateResult.Valid => statsReceiver.counter(ValidCount).incr() case PredicateResult.Invalid(reasons) => statsReceiver.counter(InvalidCount).incr() reasons.map { filterReason => statsReceiver.counter(InvalidHasReasons).incr() statsReceiver.scope(Reasons).counter(filterReason.reason).incr() } } } } /** * Helper function for timing an arrow and count individual PredicateResult */ def profilePredicateResult[Q]( predicateResult: Arrow[Q, PredicateResult], statsReceiver: StatsReceiver ): Arrow[Q, PredicateResult] = { profilePredicateResults( predicateResult.map(Seq(_)), statsReceiver ).map(_.head) } /** * Helper function for timing a stitch code block */ def profileStitchSeqResults[T]( stats: StatsReceiver )( block: => Stitch[Seq[T]] ): Stitch[Seq[T]] = { stats.counter(RequestName).incr() profileStitch(stats.stat(LatencyName), TimeUnit.MILLISECONDS) { block onSuccess { r => if (r.isEmpty) stats.counter(EmptyResultsName).incr() stats.stat(ResultsStat).add(r.size) } onFailure { e => { stats.counter(FailureName).incr() stats.scope(FailureName).counter(e.getClass.getName).incr() } } } } /** * Time a given asynchronous `f` using the given `unit`. */ def profileStitch[A](stat: Stat, unit: TimeUnit)(f: => Stitch[A]): Stitch[A] = { val start = Stopwatch.timeNanos() try { f.respond { _ => stat.add(unit.convert(Stopwatch.timeNanos() - start, TimeUnit.NANOSECONDS)) } } catch { case NonFatal(e) => stat.add(unit.convert(Stopwatch.timeNanos() - start, TimeUnit.NANOSECONDS)) Stitch.exception(e) } } def observeStitchQualityFactor[T]( stitch: Stitch[T], qualityFactorObserverOption: Option[QualityFactorObserver], statsReceiver: StatsReceiver ): Stitch[T] = { qualityFactorObserverOption .map { observer => Stitch .time(stitch) .map { case (response, stitchRunDuration) => observer(response, stitchRunDuration) val qfVal = observer.qualityFactor.currentValue.floatValue() * 10000 statsReceiver.counter(QualityFactorCounts).incr() statsReceiver .stat(QualityFactorStat) .add(qfVal) response } .lowerFromTry }.getOrElse(stitch) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/Transform.scala ================================================ package com.twitter.follow_recommendations.common.base import com.twitter.finagle.stats.StatsReceiver import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.timelines.configapi.Param /** * transform a or a list of candidate for target T * * @tparam T target type * @tparam C candidate type */ trait Transform[-T, C] { // you need to implement at least one of the two methods here. def transformItem(target: T, item: C): Stitch[C] = { transform(target, Seq(item)).map(_.head) } def transform(target: T, items: Seq[C]): Stitch[Seq[C]] def mapTarget[T2](mapper: T2 => T): Transform[T2, C] = { val original = this new Transform[T2, C] { override def transformItem(target: T2, item: C): Stitch[C] = { original.transformItem(mapper(target), item) } override def transform(target: T2, items: Seq[C]): Stitch[Seq[C]] = { original.transform(mapper(target), items) } } } /** * sequential composition. we execute this' transform first, followed by the other's transform */ def andThen[T1 <: T](other: Transform[T1, C]): Transform[T1, C] = { val original = this new Transform[T1, C] { override def transformItem(target: T1, item: C): Stitch[C] = original.transformItem(target, item).flatMap(other.transformItem(target, _)) override def transform(target: T1, items: Seq[C]): Stitch[Seq[C]] = original.transform(target, items).flatMap(other.transform(target, _)) } } def observe(statsReceiver: StatsReceiver): Transform[T, C] = { val originalTransform = this new Transform[T, C] { override def transform(target: T, items: Seq[C]): Stitch[Seq[C]] = { statsReceiver.counter(Transform.InputCandidatesCount).incr(items.size) statsReceiver.stat(Transform.InputCandidatesStat).add(items.size) StatsUtil.profileStitchSeqResults(originalTransform.transform(target, items), statsReceiver) } override def transformItem(target: T, item: C): Stitch[C] = { statsReceiver.counter(Transform.InputCandidatesCount).incr() StatsUtil.profileStitch(originalTransform.transformItem(target, item), statsReceiver) } } } } trait GatedTransform[T <: HasParams, C] extends Transform[T, C] { def gated(param: Param[Boolean]): Transform[T, C] = { val original = this (target: T, items: Seq[C]) => { if (target.params(param)) { original.transform(target, items) } else { Stitch.value(items) } } } } object Transform { val InputCandidatesCount = "input_candidates" val InputCandidatesStat = "input_candidates_stat" } class IdentityTransform[T, C] extends Transform[T, C] { override def transform(target: T, items: Seq[C]): Stitch[Seq[C]] = Stitch.value(items) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/AddressBookParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.addressbook import com.twitter.timelines.configapi.FSParam object AddressBookParams { // Used by display locations that want only to read from the ABV2 Client and ignore Manhattan // Currently the only display location that does this is the ABUploadInjection DisplayLocation object ReadFromABV2Only extends FSParam[Boolean]("addressbook_read_only_from_abv2", false) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", "src/thrift/com/twitter/hermit/usercontacts:hermit-usercontacts-scala", "strato/config/columns/onboarding/userrecs:userrecs-strato-client", "strato/src/main/scala/com/twitter/strato/client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/ForwardEmailBookSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.addressbook import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.candidate_sources.addressbook.AddressBookParams.ReadFromABV2Only import com.twitter.follow_recommendations.common.clients.addressbook.AddressbookClient import com.twitter.follow_recommendations.common.clients.addressbook.models.EdgeType import com.twitter.follow_recommendations.common.clients.addressbook.models.RecordIdentifier import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStats import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.onboarding.userrecs.ForwardEmailBookClientColumn import com.twitter.timelines.configapi.HasParams import javax.inject.Inject import javax.inject.Singleton @Singleton class ForwardEmailBookSource @Inject() ( forwardEmailBookClientColumn: ForwardEmailBookClientColumn, addressBookClient: AddressbookClient, statsReceiver: StatsReceiver = NullStatsReceiver) extends CandidateSource[HasParams with HasClientContext, CandidateUser] { override val identifier: CandidateSourceIdentifier = ForwardEmailBookSource.Identifier private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getSimpleName) /** * Generate a list of candidates for the target */ override def apply( target: HasParams with HasClientContext ): Stitch[Seq[CandidateUser]] = { val candidateUsers: Stitch[Seq[Long]] = target.getOptionalUserId .map { userId => rescueWithStats( addressBookClient.getUsers( userId = userId, identifiers = Seq(RecordIdentifier(userId = Some(userId), email = None, phoneNumber = None)), batchSize = AddressbookClient.AddressBook2BatchSize, edgeType = ForwardEmailBookSource.DefaultEdgeType, fetcherOption = if (target.params.apply(ReadFromABV2Only)) None else Some(forwardEmailBookClientColumn.fetcher), queryOption = AddressbookClient .createQueryOption( edgeType = ForwardEmailBookSource.DefaultEdgeType, isPhone = ForwardEmailBookSource.IsPhone) ), stats, "AddressBookClient" ) }.getOrElse(Stitch.Nil) candidateUsers .map( _.take(ForwardEmailBookSource.NumEmailBookEntries) .map(CandidateUser(_, score = Some(CandidateUser.DefaultCandidateScore)) .withCandidateSource(identifier))) } } object ForwardEmailBookSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.ForwardEmailBook.toString) val NumEmailBookEntries: Int = 1000 val IsPhone = false val DefaultEdgeType: EdgeType = EdgeType.Forward } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/ForwardPhoneBookSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.addressbook import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.candidate_sources.addressbook.AddressBookParams.ReadFromABV2Only import com.twitter.follow_recommendations.common.clients.addressbook.AddressbookClient import com.twitter.follow_recommendations.common.clients.addressbook.models.EdgeType import com.twitter.follow_recommendations.common.clients.addressbook.models.RecordIdentifier import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStats import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.onboarding.userrecs.ForwardPhoneContactsClientColumn import com.twitter.timelines.configapi.HasParams import javax.inject.Inject import javax.inject.Singleton @Singleton class ForwardPhoneBookSource @Inject() ( forwardPhoneContactsClientColumn: ForwardPhoneContactsClientColumn, addressBookClient: AddressbookClient, statsReceiver: StatsReceiver = NullStatsReceiver) extends CandidateSource[HasParams with HasClientContext, CandidateUser] { override val identifier: CandidateSourceIdentifier = ForwardPhoneBookSource.Identifier private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getSimpleName) /** * Generate a list of candidates for the target */ override def apply(target: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] = { val candidateUsers: Stitch[Seq[Long]] = target.getOptionalUserId .map { userId => rescueWithStats( addressBookClient.getUsers( userId, identifiers = Seq(RecordIdentifier(userId = Some(userId), email = None, phoneNumber = None)), batchSize = AddressbookClient.AddressBook2BatchSize, edgeType = ForwardPhoneBookSource.DefaultEdgeType, fetcherOption = if (target.params.apply(ReadFromABV2Only)) None else Some(forwardPhoneContactsClientColumn.fetcher), queryOption = AddressbookClient .createQueryOption( edgeType = ForwardPhoneBookSource.DefaultEdgeType, isPhone = ForwardPhoneBookSource.IsPhone) ), stats, "AddressBookClient" ) }.getOrElse(Stitch.Nil) candidateUsers .map( _.take(ForwardPhoneBookSource.NumPhoneBookEntries) .map(CandidateUser(_, score = Some(CandidateUser.DefaultCandidateScore)) .withCandidateSource(identifier))) } } object ForwardPhoneBookSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.ForwardPhoneBook.toString) val NumPhoneBookEntries: Int = 1000 val IsPhone = true val DefaultEdgeType: EdgeType = EdgeType.Forward } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/README.md ================================================ # Address Book Candidate Source Provides the accounts of a given user's forward and reverse phone and email book contacts. It is only available when the user has synced their address book with the service. ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/ReverseEmailBookSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.addressbook import com.twitter.cds.contact_consent_state.thriftscala.PurposeOfProcessing import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.clients.addressbook.AddressbookClient import com.twitter.follow_recommendations.common.clients.addressbook.models.EdgeType import com.twitter.follow_recommendations.common.clients.addressbook.models.RecordIdentifier import com.twitter.follow_recommendations.common.clients.email_storage_service.EmailStorageServiceClient import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueOptionalWithStats import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStats import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.onboarding.userrecs.ReverseEmailContactsClientColumn import com.twitter.timelines.configapi.HasParams import javax.inject.Inject import javax.inject.Singleton @Singleton class ReverseEmailBookSource @Inject() ( reverseEmailContactsClientColumn: ReverseEmailContactsClientColumn, essClient: EmailStorageServiceClient, addressBookClient: AddressbookClient, statsReceiver: StatsReceiver = NullStatsReceiver) extends CandidateSource[HasParams with HasClientContext, CandidateUser] { override val identifier: CandidateSourceIdentifier = ReverseEmailBookSource.Identifier private val rescueStats = statsReceiver.scope("ReverseEmailBookSource") /** * Generate a list of candidates for the target */ override def apply(target: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] = { val reverseCandidatesFromEmail = target.getOptionalUserId .map { userId => val verifiedEmailStitchOpt = rescueOptionalWithStats( essClient.getVerifiedEmail(userId, PurposeOfProcessing.ContentRecommendations), rescueStats, "getVerifiedEmail") verifiedEmailStitchOpt.flatMap { emailOpt => rescueWithStats( addressBookClient.getUsers( userId = userId, identifiers = emailOpt .map(email => RecordIdentifier(userId = None, email = Some(email), phoneNumber = None)).toSeq, batchSize = ReverseEmailBookSource.NumEmailBookEntries, edgeType = ReverseEmailBookSource.DefaultEdgeType, fetcherOption = if (target.params(AddressBookParams.ReadFromABV2Only)) None else Some(reverseEmailContactsClientColumn.fetcher) ), rescueStats, "AddressBookClient" ) } }.getOrElse(Stitch.Nil) reverseCandidatesFromEmail.map( _.take(ReverseEmailBookSource.NumEmailBookEntries) .map( CandidateUser(_, score = Some(CandidateUser.DefaultCandidateScore)) .withCandidateSource(identifier)) ) } } object ReverseEmailBookSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.ReverseEmailBookIbis.toString) val NumEmailBookEntries: Int = 500 val IsPhone = false val DefaultEdgeType: EdgeType = EdgeType.Reverse } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/ReversePhoneBookSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.addressbook import com.twitter.cds.contact_consent_state.thriftscala.PurposeOfProcessing import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.clients.addressbook.AddressbookClient import com.twitter.follow_recommendations.common.clients.addressbook.models.EdgeType import com.twitter.follow_recommendations.common.clients.addressbook.models.RecordIdentifier import com.twitter.follow_recommendations.common.clients.phone_storage_service.PhoneStorageServiceClient import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStats import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.onboarding.userrecs.ReversePhoneContactsClientColumn import com.twitter.timelines.configapi.HasParams import javax.inject.Inject import javax.inject.Singleton @Singleton class ReversePhoneBookSource @Inject() ( reversePhoneContactsClientColumn: ReversePhoneContactsClientColumn, pssClient: PhoneStorageServiceClient, addressBookClient: AddressbookClient, statsReceiver: StatsReceiver = NullStatsReceiver) extends CandidateSource[HasParams with HasClientContext, CandidateUser] { override val identifier: CandidateSourceIdentifier = ReversePhoneBookSource.Identifier private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getSimpleName) /** * Generate a list of candidates for the target */ override def apply(target: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] = { val reverseCandidatesFromPhones: Stitch[Seq[Long]] = target.getOptionalUserId .map { userId => pssClient .getPhoneNumbers(userId, PurposeOfProcessing.ContentRecommendations) .flatMap { phoneNumbers => rescueWithStats( addressBookClient.getUsers( userId = userId, identifiers = phoneNumbers.map(phoneNumber => RecordIdentifier(userId = None, email = None, phoneNumber = Some(phoneNumber))), batchSize = ReversePhoneBookSource.NumPhoneBookEntries, edgeType = ReversePhoneBookSource.DefaultEdgeType, fetcherOption = if (target.params(AddressBookParams.ReadFromABV2Only)) None else Some(reversePhoneContactsClientColumn.fetcher), queryOption = AddressbookClient.createQueryOption( edgeType = ReversePhoneBookSource.DefaultEdgeType, isPhone = ReversePhoneBookSource.IsPhone) ), stats, "AddressBookClient" ) } }.getOrElse(Stitch.Nil) reverseCandidatesFromPhones.map( _.take(ReversePhoneBookSource.NumPhoneBookEntries) .map( CandidateUser(_, score = Some(CandidateUser.DefaultCandidateScore)) .withCandidateSource(identifier)) ) } } object ReversePhoneBookSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.ReversePhoneBook.toString) val NumPhoneBookEntries: Int = 500 val IsPhone = true val DefaultEdgeType: EdgeType = EdgeType.Reverse } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "content-recommender/thrift/src/main/thrift:thrift-scala", "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", "src/scala/com/twitter/onboarding/relevance/features/ymbii", "strato/src/main/scala/com/twitter/strato/client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/CachedCandidateSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.base import com.twitter.escherbird.util.stitchcache.StitchCache import com.twitter.finagle.stats.StatsReceiver import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import com.twitter.util.Duration class CachedCandidateSource[K <: Object, V <: Object]( candidateSource: CandidateSource[K, V], maxCacheSize: Int, cacheTTL: Duration, statsReceiver: StatsReceiver, override val identifier: CandidateSourceIdentifier) extends CandidateSource[K, V] { private val cache = StitchCache[K, Seq[V]]( maxCacheSize = maxCacheSize, ttl = cacheTTL, statsReceiver = statsReceiver.scope(identifier.name, "cache"), underlyingCall = (k: K) => candidateSource(k) ) override def apply(target: K): Stitch[Seq[V]] = cache.readThrough(target) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/ExperimentalCandidateSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.base import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.timelines.configapi.Param import com.twitter.finagle.stats.StatsReceiver import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier /** * A wrapper of CandidateSource to make it easier to do experimentation * on new candidate generation algorithms * * @param baseSource base candidate source * @param darkreadAlgorithmParam controls whether or not to darkread candidates (fetch them even if they will not be included) * @param keepCandidatesParam controls whether or not to keep candidates from the base source * @param resultCountThresholdParam controls how many results the source must return to bucket the user and return results (greater-than-or-equal-to) * @tparam T request type. it must extend HasParams * @tparam V value type */ class ExperimentalCandidateSource[T <: HasParams, V]( baseSource: CandidateSource[T, V], darkreadAlgorithmParam: Param[Boolean], keepCandidatesParam: Param[Boolean], resultCountThresholdParam: Param[Int], baseStatsReceiver: StatsReceiver) extends CandidateSource[T, V] { override val identifier: CandidateSourceIdentifier = baseSource.identifier private[base] val statsReceiver = baseStatsReceiver.scope(s"Experimental/${identifier.name}") private[base] val requestsCounter = statsReceiver.counter("requests") private[base] val resultCountGreaterThanThresholdCounter = statsReceiver.counter("with_results_at_or_above_count_threshold") private[base] val keepResultsCounter = statsReceiver.counter("keep_results") private[base] val discardResultsCounter = statsReceiver.counter("discard_results") override def apply(request: T): Stitch[Seq[V]] = { if (request.params(darkreadAlgorithmParam)) { requestsCounter.incr() fetchFromCandidateSourceAndProcessResults(request) } else { Stitch.Nil } } private def fetchFromCandidateSourceAndProcessResults(request: T): Stitch[Seq[V]] = { baseSource(request).map { results => if (results.length >= request.params(resultCountThresholdParam)) { processResults(results, request.params(keepCandidatesParam)) } else { Nil } } } private def processResults(results: Seq[V], keepResults: Boolean): Seq[V] = { resultCountGreaterThanThresholdCounter.incr() if (keepResults) { keepResultsCounter.incr() results } else { discardResultsCounter.incr() Nil } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/RealGraphExpansionRepository.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.base import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.util.DefaultTimer import com.twitter.follow_recommendations.common.candidate_sources.base.RealGraphExpansionRepository.DefaultScore import com.twitter.follow_recommendations.common.candidate_sources.base.RealGraphExpansionRepository.MaxNumIntermediateNodesToKeep import com.twitter.follow_recommendations.common.candidate_sources.base.RealGraphExpansionRepository.FirstDegreeCandidatesTimeout import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models._ import com.twitter.onboarding.relevance.features.ymbii.ExpansionCandidateScores import com.twitter.onboarding.relevance.features.ymbii.RawYMBIICandidateFeatures import com.twitter.onboarding.relevance.store.thriftscala.CandidatesFollowedV1 import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.util.Duration import scala.collection.immutable import scala.util.control.NonFatal private final case class InterestExpansionCandidate( userID: Long, score: Double, features: RawYMBIICandidateFeatures) abstract class RealGraphExpansionRepository[Request]( realgraphExpansionStore: Fetcher[ Long, Unit, CandidatesFollowedV1 ], override val identifier: CandidateSourceIdentifier, statsReceiver: StatsReceiver = NullStatsReceiver, maxUnderlyingCandidatesToQuery: Int = 50, maxCandidatesToReturn: Int = 40, overrideUnderlyingTimeout: Option[Duration] = None, appendSocialProof: Boolean = false) extends CandidateSource[ Request, CandidateUser ] { val underlyingCandidateSource: Seq[ CandidateSource[ Request, CandidateUser ] ] private val stats = statsReceiver.scope(this.getClass.getSimpleName).scope(identifier.name) private val underlyingCandidateSourceFailureStats = stats.scope("underlying_candidate_source_failure") def apply( request: Request, ): Stitch[Seq[CandidateUser]] = { val candidatesFromUnderlyingSourcesStitch: Seq[Stitch[Seq[CandidateUser]]] = underlyingCandidateSource.map { candidateSource => candidateSource .apply(request) .within(overrideUnderlyingTimeout.getOrElse(FirstDegreeCandidatesTimeout))( DefaultTimer ) .handle { case NonFatal(e) => underlyingCandidateSourceFailureStats .counter(candidateSource.identifier.name, e.getClass.getSimpleName).incr() Seq.empty } } for { underlyingCandidatesFromEachAlgo <- Stitch.collect(candidatesFromUnderlyingSourcesStitch) // The first algorithm in the list has the highest priority. Depending on if its not // populated, fall back to other algorithms. Once a particular algorithm is chosen, only // take the top few candidates from the underlying store for expansion. underlyingCandidatesTuple = underlyingCandidatesFromEachAlgo .zip(underlyingCandidateSource) .find(_._1.nonEmpty) underlyingAlgorithmUsed: Option[CandidateSourceIdentifier] = underlyingCandidatesTuple.map { case (_, candidateSource) => candidateSource.identifier } // Take maxUnderlyingCandidatesToQuery to query realgraphExpansionStore underlyingCandidates = underlyingCandidatesTuple .map { case (candidates, candidateSource) => stats .scope("underlyingAlgorithmUsedScope").counter( candidateSource.identifier.name).incr() candidates } .getOrElse(Seq.empty) .sortBy(_.score.getOrElse(DefaultScore))(Ordering.Double.reverse) .take(maxUnderlyingCandidatesToQuery) underlyingCandidateMap: Map[Long, Double] = underlyingCandidates.map { candidate => (candidate.id, candidate.score.getOrElse(DefaultScore)) }.toMap expansionCandidates <- Stitch .traverse(underlyingCandidateMap.keySet.toSeq) { candidateId => Stitch.join( Stitch.value(candidateId), realgraphExpansionStore.fetch(candidateId).map(_.v)) }.map(_.toMap) rerankedCandidates: Seq[InterestExpansionCandidate] = rerankCandidateExpansions(underlyingCandidateMap, expansionCandidates) rerankedCandidatesFiltered = rerankedCandidates.take(maxCandidatesToReturn) } yield { rerankedCandidatesFiltered.map { candidate => val socialProofReason = if (appendSocialProof) { val socialProofIds = candidate.features.expansionCandidateScores .map(_.intermediateCandidateId) Some( Reason(Some( AccountProof(followProof = Some(FollowProof(socialProofIds, socialProofIds.size)))))) } else { None } CandidateUser( id = candidate.userID, score = Some(candidate.score), reason = socialProofReason, userCandidateSourceDetails = Some( UserCandidateSourceDetails( primaryCandidateSource = Some(identifier), candidateSourceFeatures = Map(identifier -> Seq(candidate.features)) )) ).addAddressBookMetadataIfAvailable(underlyingAlgorithmUsed.toSeq) } } } /** * Expands underlying candidates, returning them in sorted order. * * @param underlyingCandidatesMap A map from underlying candidate id to score * @param expansionCandidateMap A map from underlying candidate id to optional expansion candidates * @return A sorted sequence of expansion candidates and associated scores */ private def rerankCandidateExpansions( underlyingCandidatesMap: Map[Long, Double], expansionCandidateMap: Map[Long, Option[CandidatesFollowedV1]] ): Seq[InterestExpansionCandidate] = { // extract features val candidates: Seq[(Long, ExpansionCandidateScores)] = for { (underlyingCandidateId, underlyingCandidateScore) <- underlyingCandidatesMap.toSeq expansionCandidates = expansionCandidateMap .get(underlyingCandidateId) .flatten .map(_.candidatesFollowed) .getOrElse(Seq.empty) expansionCandidate <- expansionCandidates } yield expansionCandidate.candidateID -> ExpansionCandidateScores( underlyingCandidateId, Some(underlyingCandidateScore), Some(expansionCandidate.score) ) // merge intermediate nodes for the same candidate val dedupedCandidates: Seq[(Long, Seq[ExpansionCandidateScores])] = candidates.groupBy(_._1).mapValues(_.map(_._2).sortBy(_.intermediateCandidateId)).toSeq // score the candidate val candidatesWithTotalScore: Seq[((Long, Seq[ExpansionCandidateScores]), Double)] = dedupedCandidates.map { candidate: (Long, Seq[ExpansionCandidateScores]) => ( candidate, candidate._2.map { ieScore: ExpansionCandidateScores => ieScore.scoreFromUserToIntermediateCandidate.getOrElse(DefaultScore) * ieScore.scoreFromIntermediateToExpansionCandidate.getOrElse(DefaultScore) }.sum) } // sort candidate by score for { ((candidate, edges), score) <- candidatesWithTotalScore.sortBy(_._2)(Ordering[Double].reverse) } yield InterestExpansionCandidate( candidate, score, RawYMBIICandidateFeatures( edges.size, edges.take(MaxNumIntermediateNodesToKeep).to[immutable.Seq]) ) } } object RealGraphExpansionRepository { private val FirstDegreeCandidatesTimeout: Duration = 250.milliseconds private val MaxNumIntermediateNodesToKeep = 20 private val DefaultScore = 0.0d } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SimilarUserExpanderParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.base import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam object SimilarUserExpanderParams { case object EnableNonDirectFollowExpansion extends FSParam[Boolean]("similar_user_enable_non_direct_follow_expansion", true) case object EnableSimsExpandSeedAccountsSort extends FSParam[Boolean]("similar_user_enable_sims_expander_seed_account_sort", false) case object DefaultExpansionInputCount extends FSBoundedParam[Int]( name = "similar_user_default_expansion_input_count", default = Integer.MAX_VALUE, min = 0, max = Integer.MAX_VALUE) case object DefaultFinalCandidatesReturnedCount extends FSBoundedParam[Int]( name = "similar_user_default_final_candidates_returned_count", default = Integer.MAX_VALUE, min = 0, max = Integer.MAX_VALUE) case object DefaultEnableImplicitEngagedExpansion extends FSParam[Boolean]("similar_user_enable_implicit_engaged_expansion", true) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SimilarUserExpanderRepository.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.base import com.twitter.follow_recommendations.common.candidate_sources.base.SimilarUserExpanderParams.DefaultEnableImplicitEngagedExpansion import com.twitter.follow_recommendations.common.candidate_sources.base.SimilarUserExpanderParams.DefaultExpansionInputCount import com.twitter.follow_recommendations.common.candidate_sources.base.SimilarUserExpanderParams.DefaultFinalCandidatesReturnedCount import com.twitter.follow_recommendations.common.candidate_sources.base.SimilarUserExpanderParams.EnableNonDirectFollowExpansion import com.twitter.follow_recommendations.common.candidate_sources.base.SimilarUserExpanderParams.EnableSimsExpandSeedAccountsSort import com.twitter.follow_recommendations.common.candidate_sources.base.SimilarUserExpanderRepository.DefaultCandidateBuilder import com.twitter.follow_recommendations.common.candidate_sources.base.SimilarUserExpanderRepository.DefaultScore import com.twitter.follow_recommendations.common.models.AccountProof import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.EngagementType import com.twitter.follow_recommendations.common.models.FollowProof import com.twitter.follow_recommendations.common.models.Reason import com.twitter.follow_recommendations.common.models.SimilarToProof import com.twitter.follow_recommendations.common.models.UserCandidateSourceDetails import com.twitter.hermit.candidate.thriftscala.Candidates import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.HasParams import com.twitter.timelines.configapi.Params case class SecondDegreeCandidate(userId: Long, score: Double, socialProof: Option[Seq[Long]]) abstract class SimilarUserExpanderRepository[-Request <: HasParams]( override val identifier: CandidateSourceIdentifier, similarToCandidatesFetcher: Fetcher[ Long, Unit, Candidates ], expansionInputSizeParam: FSBoundedParam[Int] = DefaultExpansionInputCount, candidatesReturnedSizeParam: FSBoundedParam[Int] = DefaultFinalCandidatesReturnedCount, enableImplicitEngagedExpansion: FSParam[Boolean] = DefaultEnableImplicitEngagedExpansion, thresholdToAvoidExpansion: Int = 30, maxExpansionPerCandidate: Option[Int] = None, includingOriginalCandidates: Boolean = false, scorer: (Double, Double) => Double = SimilarUserExpanderRepository.DefaultScorer, aggregator: (Seq[Double]) => Double = ScoreAggregator.Max, candidateBuilder: (Long, CandidateSourceIdentifier, Double, CandidateUser) => CandidateUser = DefaultCandidateBuilder) extends TwoHopExpansionCandidateSource[ Request, CandidateUser, SecondDegreeCandidate, CandidateUser ] { val originalCandidateSource: CandidateSource[Request, CandidateUser] val backupOriginalCandidateSource: Option[CandidateSource[Request, CandidateUser]] = None override def firstDegreeNodes(request: Request): Stitch[Seq[CandidateUser]] = { val originalCandidatesStitch: Stitch[Seq[CandidateUser]] = originalCandidateSource(request) val backupCandidatesStitch: Stitch[Seq[CandidateUser]] = if (request.params(EnableNonDirectFollowExpansion)) { backupOriginalCandidateSource.map(_.apply(request)).getOrElse(Stitch.Nil) } else { Stitch.Nil } val firstDegreeCandidatesCombinedStitch: Stitch[Seq[CandidateUser]] = Stitch .join(originalCandidatesStitch, backupCandidatesStitch).map { case (firstDegreeOrigCandidates, backupFirstDegreeCandidates) => if (request.params(EnableSimsExpandSeedAccountsSort)) { firstDegreeOrigCandidates ++ backupFirstDegreeCandidates sortBy { -_.score.getOrElse(DefaultScore) } } else { firstDegreeOrigCandidates ++ backupFirstDegreeCandidates } } val candidatesAfterImplicitEngagementsRemovalStitch: Stitch[Seq[CandidateUser]] = getCandidatesAfterImplicitEngagementFiltering( request.params, firstDegreeCandidatesCombinedStitch) val firstDegreeCandidatesCombinedTrimmed = candidatesAfterImplicitEngagementsRemovalStitch.map { candidates: Seq[CandidateUser] => candidates.take(request.params(expansionInputSizeParam)) } firstDegreeCandidatesCombinedTrimmed.map { firstDegreeResults: Seq[CandidateUser] => if (firstDegreeResults.nonEmpty && firstDegreeResults.size < thresholdToAvoidExpansion) { firstDegreeResults .groupBy(_.id).mapValues( _.maxBy(_.score) ).values.toSeq } else { Nil } } } override def secondaryDegreeNodes( request: Request, firstDegreeCandidate: CandidateUser ): Stitch[Seq[SecondDegreeCandidate]] = { similarToCandidatesFetcher.fetch(firstDegreeCandidate.id).map(_.v).map { candidateListOption => candidateListOption .map { candidatesList => candidatesList.candidates.map(candidate => SecondDegreeCandidate(candidate.userId, candidate.score, candidate.socialProof)) }.getOrElse(Nil) } } override def aggregateAndScore( req: Request, firstDegreeToSecondDegreeNodesMap: Map[CandidateUser, Seq[SecondDegreeCandidate]] ): Stitch[Seq[CandidateUser]] = { val similarExpanderResults = firstDegreeToSecondDegreeNodesMap.flatMap { case (firstDegreeCandidate, seqOfSecondDegreeCandidates) => val sourceScore = firstDegreeCandidate.score.getOrElse(DefaultScore) val results: Seq[CandidateUser] = seqOfSecondDegreeCandidates.map { secondDegreeCandidate => val score = scorer(sourceScore, secondDegreeCandidate.score) candidateBuilder(secondDegreeCandidate.userId, identifier, score, firstDegreeCandidate) } maxExpansionPerCandidate match { case None => results case Some(limit) => results.sortBy(-_.score.getOrElse(DefaultScore)).take(limit) } }.toSeq val allCandidates = { if (includingOriginalCandidates) firstDegreeToSecondDegreeNodesMap.keySet.toSeq else Nil } ++ similarExpanderResults val groupedCandidates: Seq[CandidateUser] = allCandidates .groupBy(_.id) .flatMap { case (_, candidates) => val finalScore = aggregator(candidates.map(_.score.getOrElse(DefaultScore))) val candidateSourceDetailsCombined = aggregateCandidateSourceDetails(candidates) val accountSocialProofcombined = aggregateAccountSocialProof(candidates) candidates.headOption.map( _.copy( score = Some(finalScore), reason = accountSocialProofcombined, userCandidateSourceDetails = candidateSourceDetailsCombined) .withCandidateSource(identifier)) } .toSeq Stitch.value( groupedCandidates .sortBy { -_.score.getOrElse(DefaultScore) }.take(req.params(candidatesReturnedSizeParam)) ) } def aggregateCandidateSourceDetails( candidates: Seq[CandidateUser] ): Option[UserCandidateSourceDetails] = { candidates .map { candidate => candidate.userCandidateSourceDetails.map(_.candidateSourceScores).getOrElse(Map.empty) }.reduceLeftOption { (scoreMap1, scoreMap2) => scoreMap1 ++ scoreMap2 }.map { UserCandidateSourceDetails(primaryCandidateSource = None, _) } } def aggregateAccountSocialProof(candidates: Seq[CandidateUser]): Option[Reason] = { candidates .map { candidate => ( candidate.reason .flatMap(_.accountProof.flatMap(_.similarToProof.map(_.similarTo))).getOrElse(Nil), candidate.reason .flatMap(_.accountProof.flatMap(_.followProof.map(_.followedBy))).getOrElse(Nil), candidate.reason .flatMap(_.accountProof.flatMap(_.followProof.map(_.numIds))).getOrElse(0) ) }.reduceLeftOption { (accountProofOne, accountProofTwo) => ( // merge similarToIds accountProofOne._1 ++ accountProofTwo._1, // merge followedByIds accountProofOne._2 ++ accountProofTwo._2, // add numIds accountProofOne._3 + accountProofTwo._3) }.map { proofs => Reason(accountProof = Some( AccountProof( similarToProof = Some(SimilarToProof(proofs._1)), followProof = if (proofs._2.nonEmpty) Some(FollowProof(proofs._2, proofs._3)) else None ))) } } def getCandidatesAfterImplicitEngagementFiltering( params: Params, firstDegreeCandidatesStitch: Stitch[Seq[CandidateUser]] ): Stitch[Seq[CandidateUser]] = { if (!params(enableImplicitEngagedExpansion)) { /** * Remove candidates whose engagement types only contain implicit engagements * (e.g. Profile View, Tweet Click) and only expand those candidates who contain explicit * engagements. */ firstDegreeCandidatesStitch.map { candidates => candidates.filter { cand => cand.engagements.exists(engage => engage == EngagementType.Like || engage == EngagementType.Retweet || engage == EngagementType.Mention) } } } else { firstDegreeCandidatesStitch } } } object SimilarUserExpanderRepository { val DefaultScorer: (Double, Double) => Double = (sourceScore: Double, similarScore: Double) => similarScore val MultiplyScorer: (Double, Double) => Double = (sourceScore: Double, similarScore: Double) => sourceScore * similarScore val SourceScorer: (Double, Double) => Double = (sourceScore: Double, similarScore: Double) => sourceScore val DefaultScore = 0.0d val DefaultCandidateBuilder: ( Long, CandidateSourceIdentifier, Double, CandidateUser ) => CandidateUser = ( userId: Long, _: CandidateSourceIdentifier, score: Double, candidate: CandidateUser ) => { val originalCandidateSourceDetails = candidate.userCandidateSourceDetails.flatMap { candSourceDetails => candSourceDetails.primaryCandidateSource.map { primaryCandidateSource => UserCandidateSourceDetails( primaryCandidateSource = None, candidateSourceScores = Map(primaryCandidateSource -> candidate.score)) } } CandidateUser( id = userId, score = Some(score), userCandidateSourceDetails = originalCandidateSourceDetails, reason = Some(Reason(Some(AccountProof(similarToProof = Some(SimilarToProof(Seq(candidate.id))))))) ) } val FollowClusterCandidateBuilder: ( Long, CandidateSourceIdentifier, Double, CandidateUser ) => CandidateUser = (userId: Long, _: CandidateSourceIdentifier, score: Double, candidate: CandidateUser) => { val originalCandidateSourceDetails = candidate.userCandidateSourceDetails.flatMap { candSourceDetails => candSourceDetails.primaryCandidateSource.map { primaryCandidateSource => UserCandidateSourceDetails( primaryCandidateSource = None, candidateSourceScores = Map(primaryCandidateSource -> candidate.score)) } } val originalFollowCluster = candidate.reason .flatMap(_.accountProof.flatMap(_.followProof.map(_.followedBy))) CandidateUser( id = userId, score = Some(score), userCandidateSourceDetails = originalCandidateSourceDetails, reason = Some( Reason( Some( AccountProof( similarToProof = Some(SimilarToProof(Seq(candidate.id))), followProof = originalFollowCluster.map(follows => FollowProof(follows, follows.size))))) ) ) } } object ScoreAggregator { // aggregate the same candidates with same id by taking the one with largest score val Max: Seq[Double] => Double = (candidateScores: Seq[Double]) => { candidateScores.max } // aggregate the same candidates with same id by taking the sum of the scores val Sum: Seq[Double] => Double = (candidateScores: Seq[Double]) => { candidateScores.sum } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SocialProofEnforcedCandidateSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.base import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.transforms.modify_social_proof.ModifySocialProof import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.util.Duration abstract class SocialProofEnforcedCandidateSource( candidateSource: CandidateSource[HasClientContext with HasParams, CandidateUser], modifySocialProof: ModifySocialProof, minNumSocialProofsRequired: Int, override val identifier: CandidateSourceIdentifier, baseStatsReceiver: StatsReceiver) extends CandidateSource[HasClientContext with HasParams, CandidateUser] { val statsReceiver = baseStatsReceiver.scope(identifier.name) override def apply(target: HasClientContext with HasParams): Stitch[Seq[CandidateUser]] = { val mustCallSgs: Boolean = target.params(SocialProofEnforcedCandidateSourceParams.MustCallSgs) val callSgsCachedColumn: Boolean = target.params(SocialProofEnforcedCandidateSourceParams.CallSgsCachedColumn) val QueryIntersectionIdsNum: Int = target.params(SocialProofEnforcedCandidateSourceParams.QueryIntersectionIdsNum) val MaxNumCandidatesToAnnotate: Int = target.params(SocialProofEnforcedCandidateSourceParams.MaxNumCandidatesToAnnotate) val gfsIntersectionIdsNum: Int = target.params(SocialProofEnforcedCandidateSourceParams.GfsIntersectionIdsNum) val sgsIntersectionIdsNum: Int = target.params(SocialProofEnforcedCandidateSourceParams.SgsIntersectionIdsNum) val gfsLagDuration: Duration = target.params(SocialProofEnforcedCandidateSourceParams.GfsLagDurationInDays) candidateSource(target) .flatMap { candidates => val candidatesWithoutEnoughSocialProof = candidates .collect { case candidate if !candidate.followedBy.exists(_.size >= minNumSocialProofsRequired) => candidate } statsReceiver .stat("candidates_with_no_social_proofs").add(candidatesWithoutEnoughSocialProof.size) val candidatesToAnnotate = candidatesWithoutEnoughSocialProof.take(MaxNumCandidatesToAnnotate) statsReceiver.stat("candidates_to_annotate").add(candidatesToAnnotate.size) val annotatedCandidatesMapStitch = target.getOptionalUserId .map { userId => modifySocialProof .hydrateSocialProof( userId, candidatesToAnnotate, Some(QueryIntersectionIdsNum), mustCallSgs, callSgsCachedColumn, gfsLagDuration = gfsLagDuration, gfsIntersectionIds = gfsIntersectionIdsNum, sgsIntersectionIds = sgsIntersectionIdsNum ).map { annotatedCandidates => annotatedCandidates .map(annotatedCandidate => (annotatedCandidate.id, annotatedCandidate)).toMap } }.getOrElse(Stitch.value(Map.empty[Long, CandidateUser])) annotatedCandidatesMapStitch.map { annotatedCandidatesMap => candidates .flatMap { candidate => if (candidate.followedBy.exists(_.size >= minNumSocialProofsRequired)) { Some(candidate) } else { annotatedCandidatesMap.get(candidate.id).collect { case annotatedCandidate if annotatedCandidate.followedBy.exists( _.size >= minNumSocialProofsRequired) => annotatedCandidate } } }.map(_.withCandidateSource(identifier)) } } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SocialProofEnforcedCandidateSourceFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.base import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.timelines.configapi.Param import com.twitter.util.Duration import javax.inject.Inject import javax.inject.Singleton @Singleton class SocialProofEnforcedCandidateSourceFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( SocialProofEnforcedCandidateSourceParams.MustCallSgs, SocialProofEnforcedCandidateSourceParams.CallSgsCachedColumn, ) override val intFSParams: Seq[FSBoundedParam[Int]] = Seq( SocialProofEnforcedCandidateSourceParams.QueryIntersectionIdsNum, SocialProofEnforcedCandidateSourceParams.MaxNumCandidatesToAnnotate, SocialProofEnforcedCandidateSourceParams.GfsIntersectionIdsNum, SocialProofEnforcedCandidateSourceParams.SgsIntersectionIdsNum, ) override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( SocialProofEnforcedCandidateSourceParams.GfsLagDurationInDays ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SocialProofEnforcedCandidateSourceParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.base import com.twitter.conversions.DurationOps._ import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.util.Duration object SocialProofEnforcedCandidateSourceParams { case object MustCallSgs extends FSParam[Boolean]("social_proof_enforced_candidate_source_must_call_sgs", true) case object CallSgsCachedColumn extends FSParam[Boolean]( "social_proof_enforced_candidate_source_call_sgs_cached_column", false) case object QueryIntersectionIdsNum extends FSBoundedParam[Int]( name = "social_proof_enforced_candidate_source_query_intersection_ids_num", default = 3, min = 0, max = Integer.MAX_VALUE) case object MaxNumCandidatesToAnnotate extends FSBoundedParam[Int]( name = "social_proof_enforced_candidate_source_max_num_candidates_to_annotate", default = 50, min = 0, max = Integer.MAX_VALUE) case object GfsIntersectionIdsNum extends FSBoundedParam[Int]( name = "social_proof_enforced_candidate_source_gfs_intersection_ids_num", default = 3, min = 0, max = Integer.MAX_VALUE) case object SgsIntersectionIdsNum extends FSBoundedParam[Int]( name = "social_proof_enforced_candidate_source_sgs_intersection_ids_num", default = 10, min = 0, max = Integer.MAX_VALUE) case object GfsLagDurationInDays extends FSBoundedParam[Duration]( name = "social_proof_enforced_candidate_source_gfs_lag_duration_in_days", default = 14.days, min = 1.days, max = 60.days) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromDays } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/StratoFetcherSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.base import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher abstract class StratoFetcherSource[K, U, V]( fetcher: Fetcher[K, U, V], view: U, override val identifier: CandidateSourceIdentifier) extends CandidateSource[K, CandidateUser] { def map(user: K, v: V): Seq[CandidateUser] override def apply(target: K): Stitch[Seq[CandidateUser]] = { fetcher .fetch(target, view) .map { result => result.v .map { candidates => map(target, candidates) } .getOrElse(Nil) .map(_.withCandidateSource(identifier)) } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/StratoFetcherWithUnitViewSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.base import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.strato.client.Fetcher abstract class StratoFetcherWithUnitViewSource[K, V]( fetcher: Fetcher[K, Unit, V], override val identifier: CandidateSourceIdentifier) extends StratoFetcherSource[K, Unit, V](fetcher, Unit, identifier) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/TweetAuthorsCandidateSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.base import com.twitter.follow_recommendations.common.models.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.stitch.Stitch /** * base trait for tweet authors based algorithms, e.g. topical tweet authors, twistly, ... * * @tparam Target target type * @tparam Candidate output candidate types */ trait TweetAuthorsCandidateSource[-Target, +Candidate] extends CandidateSource[Target, Candidate] { /** * fetch Tweet candidates */ def getTweetCandidates(target: Target): Stitch[Seq[TweetCandidate]] /** * fetch authorId */ def getTweetAuthorId(tweetCandidate: TweetCandidate): Stitch[Option[Long]] /** * wrap candidate ID and TweetAuthorProof in Candidate */ def toCandidate(authorId: Long, tweetIds: Seq[Long], score: Option[Double]): Candidate /** * aggregate scores, default to the first score */ def aggregator(scores: Seq[Double]): Double = scores.headOption.getOrElse(TweetAuthorsCandidateSource.DefaultScore) /** * aggregation method for a group of tweet candidates */ def aggregateAndScore( target: Target, tweetCandidates: Seq[TweetCandidate] ): Seq[Candidate] /** * generate a list of candidates for the target */ def build( target: Target ): Stitch[Seq[Candidate]] = { // Fetch Tweet candidates and hydrate author IDs val tweetCandidatesStitch = for { tweetCandidates <- getTweetCandidates(target) authorIds <- Stitch.collect(tweetCandidates.map(getTweetAuthorId(_))) } yield { for { (authorIdOpt, tweetCandidate) <- authorIds.zip(tweetCandidates) authorId <- authorIdOpt } yield tweetCandidate.copy(authorId = authorId) } // Aggregate and score, convert to candidate tweetCandidatesStitch.map(aggregateAndScore(target, _)) } def apply(target: Target): Stitch[Seq[Candidate]] = build(target) } object TweetAuthorsCandidateSource { final val DefaultScore: Double = 0.0 } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/TwoHopExpansionCandidateSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.base import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.stitch.Stitch /** * base trait for two-hop expansion based algorithms, e.g. online_stp, phonebook_prediction, * recent following sims, recent engagement sims, ... * * @tparam Target target type * @tparam FirstDegree type of first degree nodes * @tparam SecondaryDegree type of secondary degree nodes * @tparam Candidate output candidate types */ trait TwoHopExpansionCandidateSource[-Target, FirstDegree, SecondaryDegree, +Candidate] extends CandidateSource[Target, Candidate] { /** * fetch first degree nodes given request */ def firstDegreeNodes(req: Target): Stitch[Seq[FirstDegree]] /** * fetch secondary degree nodes given request and first degree nodes */ def secondaryDegreeNodes(req: Target, node: FirstDegree): Stitch[Seq[SecondaryDegree]] /** * aggregate and score the candidates to generate final results */ def aggregateAndScore( req: Target, firstDegreeToSecondDegreeNodesMap: Map[FirstDegree, Seq[SecondaryDegree]] ): Stitch[Seq[Candidate]] /** * Generate a list of candidates for the target */ def apply(target: Target): Stitch[Seq[Candidate]] = { for { firstDegreeNodes <- firstDegreeNodes(target) secondaryDegreeNodes <- Stitch.traverse(firstDegreeNodes)(secondaryDegreeNodes(target, _)) aggregated <- aggregateAndScore(target, firstDegreeNodes.zip(secondaryDegreeNodes).toMap) } yield aggregated } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "hermit/hermit-core/src/main/scala/com/twitter/hermit/model", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", "src/thrift/com/twitter/onboarding/relevance/crowd_search_accounts:crowd_search_accounts-scala", "strato/config/columns/onboarding/userrecs:userrecs-strato-client", "strato/src/main/scala/com/twitter/strato/client", "util/util-core/src/main/scala/com/twitter/conversions", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/CrowdSearchAccountsFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.Param import javax.inject.Inject import javax.inject.Singleton @Singleton class CrowdSearchAccountsFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( CrowdSearchAccountsParams.CandidateSourceEnabled, ) override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( CrowdSearchAccountsParams.CandidateSourceWeight, ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/CrowdSearchAccountsParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSEnumSeqParam import com.twitter.timelines.configapi.FSParam object CrowdSearchAccountsParams { // whether or not to fetch CrowdSearchAccounts candidate sources case object CandidateSourceEnabled extends FSParam[Boolean]("crowd_search_accounts_candidate_source_enabled", false) /** * Contains the logic key for account filtering and ranking. Currently we have 3 main logic keys * - new_daily: filtering top searched accounts with max daily searches based on new users * - new_weekly: filtering top searched accounts with max weekly searches based on new users * - daily: filtering top searched accounts with max daily searches * - weekly: filtering top searched accounts with max weekly searches * Mapping of the Logic Id to Logic key is done via @enum AccountsFilteringAndRankingLogic */ case object AccountsFilteringAndRankingLogics extends FSEnumSeqParam[AccountsFilteringAndRankingLogicId.type]( name = "crowd_search_accounts_filtering_and_ranking_logic_ids", default = Seq(AccountsFilteringAndRankingLogicId.SearchesWeekly), enum = AccountsFilteringAndRankingLogicId) case object CandidateSourceWeight extends FSBoundedParam[Double]( "crowd_search_accounts_candidate_source_weight", default = 1200, min = 0.001, max = 2000) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/CrowdSearchAccountsSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts import com.twitter.escherbird.util.stitchcache.StitchCache import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsParams.AccountsFilteringAndRankingLogics import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsParams.CandidateSourceEnabled import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasGeohashAndCountryCode import com.twitter.hermit.model.Algorithm import com.twitter.onboarding.relevance.crowd_search_accounts.thriftscala.CrowdSearchAccounts import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.onboarding.userrecs.CrowdSearchAccountsClientColumn import com.twitter.timelines.configapi.HasParams import com.twitter.util.Duration import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton object AccountsFilteringAndRankingLogicId extends Enumeration { type AccountsFilteringAndRankingLogicId = Value val NewSearchesDaily: AccountsFilteringAndRankingLogicId = Value("new_searches_daily") val NewSearchesWeekly: AccountsFilteringAndRankingLogicId = Value("new_searches_weekly") val SearchesDaily: AccountsFilteringAndRankingLogicId = Value("searches_daily") val SearchesWeekly: AccountsFilteringAndRankingLogicId = Value("searches_weekly") } object CrowdSearchAccountsSource { val MaxCacheSize = 500 val CacheTTL: Duration = Duration.fromHours(24) type Target = HasParams with HasClientContext with HasGeohashAndCountryCode val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.CrowdSearchAccounts.toString) } @Singleton class CrowdSearchAccountsSource @Inject() ( crowdSearchAccountsClientColumn: CrowdSearchAccountsClientColumn, statsReceiver: StatsReceiver, ) extends CandidateSource[CrowdSearchAccountsSource.Target, CandidateUser] with Logging { /** @see [[CandidateSourceIdentifier]] */ override val identifier: CandidateSourceIdentifier = CrowdSearchAccountsSource.Identifier private val stats = statsReceiver.scope(identifier.name) private val requestsStats = stats.counter("requests") private val noCountryCodeStats = stats.counter("no_country_code") private val successStats = stats.counter("success") private val errorStats = stats.counter("error") private val cache = StitchCache[String, Option[CrowdSearchAccounts]]( maxCacheSize = CrowdSearchAccountsSource.MaxCacheSize, ttl = CrowdSearchAccountsSource.CacheTTL, statsReceiver = statsReceiver.scope(identifier.name, "cache"), underlyingCall = (k: String) => { crowdSearchAccountsClientColumn.fetcher .fetch(k) .map { result => result.v } } ) /** returns a Seq of ''potential'' content */ override def apply( target: CrowdSearchAccountsSource.Target ): Stitch[Seq[CandidateUser]] = { if (!target.params(CandidateSourceEnabled)) { return Stitch.value(Seq[CandidateUser]()) } requestsStats.incr() target.getCountryCode .orElse(target.geohashAndCountryCode.flatMap(_.countryCode)).map { countryCode => Stitch .collect(target .params(AccountsFilteringAndRankingLogics).map(logic => cache.readThrough(countryCode.toUpperCase() + "-" + logic))) .onSuccess(_ => { successStats.incr() }) .onFailure(t => { debug("candidate source failed identifier = %s".format(identifier), t) errorStats.incr() }) .map(transformCrowdSearchAccountsToCandidateSource) }.getOrElse { noCountryCodeStats.incr() Stitch.value(Seq[CandidateUser]()) } } private def transformCrowdSearchAccountsToCandidateSource( crowdSearchAccounts: Seq[Option[CrowdSearchAccounts]] ): Seq[CandidateUser] = { crowdSearchAccounts .flatMap(opt => opt .map(accounts => accounts.accounts.map(account => CandidateUser( id = account.accountId, score = Some(account.searchActivityScore), ).withCandidateSource(identifier))) .getOrElse(Seq[CandidateUser]())) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/README.md ================================================ # Crowd Search Candidate Source Provides the most searched accounts within a specific country over the past 1 and 7 days. * When we refer to "most searched accounts", we are referring to accounts that have been clicked on the most frequently by users after they see search results in both the typeahead and search results page. * The results returned by the service have undergone health filters. ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "src/thrift/com/twitter/hermit/pop_geo:hermit-pop-geo-scala", "strato/config/columns/onboarding/userrecs:userrecs-strato-client", "strato/src/main/scala/com/twitter/strato/client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/BasePopGeoHashSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.geo import com.google.inject.Singleton import com.twitter.finagle.stats.Counter import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasGeohashAndCountryCode import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import javax.inject.Inject @Singleton class BasePopGeohashSource @Inject() ( popGeoSource: CandidateSource[String, CandidateUser], statsReceiver: StatsReceiver) extends CandidateSource[ HasParams with HasClientContext with HasGeohashAndCountryCode, CandidateUser ] with BasePopGeohashSourceConfig { val stats: StatsReceiver = statsReceiver // counter to check if we found a geohash value in the request val foundGeohashCounter: Counter = stats.counter("found_geohash_value") // counter to check if we are missing a geohash value in the request val missingGeohashCounter: Counter = stats.counter("missing_geohash_value") /** @see [[CandidateSourceIdentifier]] */ override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( "BasePopGeohashSource") override def apply( target: HasParams with HasClientContext with HasGeohashAndCountryCode ): Stitch[Seq[CandidateUser]] = { if (!candidateSourceEnabled(target)) { return Stitch.Nil } target.geohashAndCountryCode .flatMap(_.geohash).map { geohash => foundGeohashCounter.incr() val keys = (minGeohashLength(target) to math.min(maxGeohashLength(target), geohash.length)) .map("geohash_" + geohash.take(_)).reverse if (returnResultFromAllPrecision(target)) { Stitch .collect(keys.map(popGeoSource.apply)).map( _.flatten.map(_.withCandidateSource(identifier)) ) } else { Stitch .collect(keys.map(popGeoSource.apply)).map( _.find(_.nonEmpty) .getOrElse(Nil) .take(maxResults(target)).map(_.withCandidateSource(identifier)) ) } }.getOrElse { missingGeohashCounter.incr() Stitch.Nil } } } trait BasePopGeohashSourceConfig { type Target = HasParams with HasClientContext def maxResults(target: Target): Int = 200 def minGeohashLength(target: Target): Int = 2 def maxGeohashLength(target: Target): Int = 4 def returnResultFromAllPrecision(target: Target): Boolean = false def candidateSourceEnabled(target: Target): Boolean = false } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopCountryBackFillSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.geo import com.google.inject.Singleton import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import javax.inject.Inject @Singleton class PopCountryBackFillSource @Inject() (popGeoSource: PopGeoSource) extends CandidateSource[HasClientContext with HasParams, CandidateUser] { override val identifier: CandidateSourceIdentifier = PopCountryBackFillSource.Identifier override def apply(target: HasClientContext with HasParams): Stitch[Seq[CandidateUser]] = { target.getOptionalUserId .map(_ => popGeoSource(PopCountryBackFillSource.DefaultKey) .map(_.take(PopCountryBackFillSource.MaxResults).map(_.withCandidateSource(identifier)))) .getOrElse(Stitch.Nil) } } object PopCountryBackFillSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier(Algorithm.PopCountryBackFill.toString) val MaxResults = 40 val DefaultKey = "country_US" } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopCountrySource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.geo import com.google.inject.Singleton import com.twitter.core_workflows.user_model.thriftscala.UserState import com.twitter.finagle.stats.Counter import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasGeohashAndCountryCode import com.twitter.follow_recommendations.common.models.HasUserState import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import javax.inject.Inject @Singleton class PopCountrySource @Inject() ( popGeoSource: PopGeoSource, statsReceiver: StatsReceiver) extends CandidateSource[ HasClientContext with HasParams with HasUserState with HasGeohashAndCountryCode, CandidateUser ] { override val identifier: CandidateSourceIdentifier = PopCountrySource.Identifier val stats: StatsReceiver = statsReceiver.scope("PopCountrySource") // counter to check if we found a country code value in the request val foundCountryCodeCounter: Counter = stats.counter("found_country_code_value") // counter to check if we are missing a country code value in the request val missingCountryCodeCounter: Counter = stats.counter("missing_country_code_value") override def apply( target: HasClientContext with HasParams with HasUserState with HasGeohashAndCountryCode ): Stitch[Seq[CandidateUser]] = { target.geohashAndCountryCode .flatMap(_.countryCode).map { countryCode => foundCountryCodeCounter.incr() if (target.userState.exists(PopCountrySource.BlacklistedTargetUserStates.contains)) { Stitch.Nil } else { popGeoSource("country_" + countryCode) .map(_.take(PopCountrySource.MaxResults).map(_.withCandidateSource(identifier))) } }.getOrElse { missingCountryCodeCounter.incr() Stitch.Nil } } } object PopCountrySource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.PopCountry.toString) val MaxResults = 40 val BlacklistedTargetUserStates: Set[UserState] = Set( UserState.HeavyTweeter, UserState.HeavyNonTweeter, UserState.MediumTweeter, UserState.MediumNonTweeter) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoQualityFollowSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.geo import com.google.inject.Singleton import com.twitter.escherbird.util.stitchcache.StitchCache import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.models.AccountProof import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.PopularInGeoProof import com.twitter.follow_recommendations.common.models.Reason import com.twitter.hermit.model.Algorithm import com.twitter.hermit.pop_geo.thriftscala.PopUsersInPlace import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.onboarding.userrecs.UniquePopQualityFollowUsersInPlaceClientColumn import com.twitter.util.Duration import javax.inject.Inject @Singleton class PopGeohashQualityFollowSource @Inject() ( popGeoSource: PopGeoQualityFollowSource, statsReceiver: StatsReceiver) extends BasePopGeohashSource( popGeoSource = popGeoSource, statsReceiver = statsReceiver.scope("PopGeohashQualityFollowSource"), ) { override val identifier: CandidateSourceIdentifier = PopGeohashQualityFollowSource.Identifier override def maxResults(target: Target): Int = { target.params(PopGeoQualityFollowSourceParams.PopGeoSourceMaxResultsPerPrecision) } override def minGeohashLength(target: Target): Int = { target.params(PopGeoQualityFollowSourceParams.PopGeoSourceGeoHashMinPrecision) } override def maxGeohashLength(target: Target): Int = { target.params(PopGeoQualityFollowSourceParams.PopGeoSourceGeoHashMaxPrecision) } override def returnResultFromAllPrecision(target: Target): Boolean = { target.params(PopGeoQualityFollowSourceParams.PopGeoSourceReturnFromAllPrecisions) } override def candidateSourceEnabled(target: Target): Boolean = { target.params(PopGeoQualityFollowSourceParams.CandidateSourceEnabled) } } object PopGeohashQualityFollowSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.PopGeohashQualityFollow.toString) } object PopGeoQualityFollowSource { val MaxCacheSize = 20000 val CacheTTL: Duration = Duration.fromHours(24) val MaxResults = 200 } @Singleton class PopGeoQualityFollowSource @Inject() ( popGeoQualityFollowClientColumn: UniquePopQualityFollowUsersInPlaceClientColumn, statsReceiver: StatsReceiver, ) extends CandidateSource[String, CandidateUser] { /** @see [[CandidateSourceIdentifier]] */ override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( "PopGeoQualityFollowSource") private val cache = StitchCache[String, Option[PopUsersInPlace]]( maxCacheSize = PopGeoQualityFollowSource.MaxCacheSize, ttl = PopGeoQualityFollowSource.CacheTTL, statsReceiver = statsReceiver.scope(identifier.name, "cache"), underlyingCall = (k: String) => { popGeoQualityFollowClientColumn.fetcher .fetch(k) .map { result => result.v } } ) override def apply(target: String): Stitch[Seq[CandidateUser]] = { val result: Stitch[Option[PopUsersInPlace]] = cache.readThrough(target) result.map { pu => pu.map { candidates => candidates.popUsers.sortBy(-_.score).take(PopGeoQualityFollowSource.MaxResults).map { candidate => CandidateUser( id = candidate.userId, score = Some(candidate.score), reason = Some( Reason( Some( AccountProof( popularInGeoProof = Some(PopularInGeoProof(location = candidates.place)) ) ) ) ) ) } }.getOrElse(Nil) } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoQualityFollowSourceFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.geo import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @Singleton class PopGeoQualityFollowSourceFSConfig @Inject() () extends FeatureSwitchConfig { override val intFSParams: Seq[FSBoundedParam[Int] with FSName] = Seq( PopGeoQualityFollowSourceParams.PopGeoSourceGeoHashMaxPrecision, PopGeoQualityFollowSourceParams.PopGeoSourceGeoHashMinPrecision, PopGeoQualityFollowSourceParams.PopGeoSourceMaxResultsPerPrecision ) override val doubleFSParams: Seq[FSBoundedParam[Double] with FSName] = Seq( PopGeoQualityFollowSourceParams.CandidateSourceWeight ) override val booleanFSParams: Seq[FSParam[Boolean] with FSName] = Seq( PopGeoQualityFollowSourceParams.CandidateSourceEnabled, PopGeoQualityFollowSourceParams.PopGeoSourceReturnFromAllPrecisions ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoQualityFollowSourceParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.geo import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam object PopGeoQualityFollowSourceParams { case object CandidateSourceEnabled extends FSParam[Boolean]("pop_geo_quality_follow_source_enabled", false) case object PopGeoSourceGeoHashMinPrecision extends FSBoundedParam[Int]( "pop_geo_quality_follow_source_geo_hash_min_precision", default = 2, min = 0, max = 10) case object PopGeoSourceGeoHashMaxPrecision extends FSBoundedParam[Int]( "pop_geo_quality_follow_source_geo_hash_max_precision", default = 3, min = 0, max = 10) case object PopGeoSourceReturnFromAllPrecisions extends FSParam[Boolean]( "pop_geo_quality_follow_source_return_from_all_precisions", default = false) case object PopGeoSourceMaxResultsPerPrecision extends FSBoundedParam[Int]( "pop_geo_quality_follow_source_max_results_per_precision", default = 200, min = 0, max = 1000) case object CandidateSourceWeight extends FSBoundedParam[Double]( "pop_geo_quality_follow_source_weight", default = 200, min = 0.001, max = 2000) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.geo import com.google.inject.Singleton import com.google.inject.name.Named import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.candidate_sources.base.CachedCandidateSource import com.twitter.follow_recommendations.common.candidate_sources.base.StratoFetcherWithUnitViewSource import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.follow_recommendations.common.models.AccountProof import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.PopularInGeoProof import com.twitter.follow_recommendations.common.models.Reason import com.twitter.hermit.pop_geo.thriftscala.PopUsersInPlace import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.strato.client.Fetcher import com.twitter.util.Duration import javax.inject.Inject @Singleton class BasePopGeoSource @Inject() ( @Named(GuiceNamedConstants.POP_USERS_IN_PLACE_FETCHER) fetcher: Fetcher[ String, Unit, PopUsersInPlace ]) extends StratoFetcherWithUnitViewSource[String, PopUsersInPlace]( fetcher, BasePopGeoSource.Identifier) { override def map(target: String, candidates: PopUsersInPlace): Seq[CandidateUser] = BasePopGeoSource.map(target, candidates) } object BasePopGeoSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("BasePopGeoSource") val MaxResults = 200 def map(target: String, candidates: PopUsersInPlace): Seq[CandidateUser] = candidates.popUsers.sortBy(-_.score).take(BasePopGeoSource.MaxResults).view.map { candidate => CandidateUser( id = candidate.userId, score = Some(candidate.score), reason = Some( Reason( Some( AccountProof( popularInGeoProof = Some(PopularInGeoProof(location = candidates.place)) ) ) ) ) ) } } @Singleton class PopGeoSource @Inject() (basePopGeoSource: BasePopGeoSource, statsReceiver: StatsReceiver) extends CachedCandidateSource[String, CandidateUser]( basePopGeoSource, PopGeoSource.MaxCacheSize, PopGeoSource.CacheTTL, statsReceiver, PopGeoSource.Identifier) object PopGeoSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("PopGeoSource") val MaxCacheSize = 20000 val CacheTTL: Duration = 1.hours } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoSourceFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.geo import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @Singleton class PopGeoSourceFSConfig @Inject() () extends FeatureSwitchConfig { override val intFSParams: Seq[FSBoundedParam[Int] with FSName] = Seq( PopGeoSourceParams.PopGeoSourceGeoHashMaxPrecision, PopGeoSourceParams.PopGeoSourceMaxResultsPerPrecision, PopGeoSourceParams.PopGeoSourceGeoHashMinPrecision, ) override val booleanFSParams: Seq[FSParam[Boolean] with FSName] = Seq( PopGeoSourceParams.PopGeoSourceReturnFromAllPrecisions, ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoSourceParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.geo import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam object PopGeoSourceParams { case object PopGeoSourceGeoHashMinPrecision extends FSBoundedParam[Int]( "pop_geo_source_geo_hash_min_precision", default = 2, min = 0, max = 10) case object PopGeoSourceGeoHashMaxPrecision extends FSBoundedParam[Int]( "pop_geo_source_geo_hash_max_precision", default = 4, min = 0, max = 10) case object PopGeoSourceReturnFromAllPrecisions extends FSParam[Boolean]("pop_geo_source_return_from_all_precisions", default = false) case object PopGeoSourceMaxResultsPerPrecision extends FSBoundedParam[Int]( "pop_geo_source_max_results_per_precision", default = 200, min = 0, max = 1000) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeohashSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.geo import com.google.inject.Singleton import com.twitter.finagle.stats.StatsReceiver import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import javax.inject.Inject @Singleton class PopGeohashSource @Inject() ( popGeoSource: PopGeoSource, statsReceiver: StatsReceiver) extends BasePopGeohashSource( popGeoSource = popGeoSource, statsReceiver = statsReceiver.scope("PopGeohashSource"), ) { override def candidateSourceEnabled(target: Target): Boolean = true override val identifier: CandidateSourceIdentifier = PopGeohashSource.Identifier override def minGeohashLength(target: Target): Int = { target.params(PopGeoSourceParams.PopGeoSourceGeoHashMinPrecision) } override def maxResults(target: Target): Int = { target.params(PopGeoSourceParams.PopGeoSourceMaxResultsPerPrecision) } override def maxGeohashLength(target: Target): Int = { target.params(PopGeoSourceParams.PopGeoSourceGeoHashMaxPrecision) } override def returnResultFromAllPrecision(target: Target): Boolean = { target.params(PopGeoSourceParams.PopGeoSourceReturnFromAllPrecisions) } } object PopGeohashSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.PopGeohash.toString) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/README.md ================================================ # Pop Geo Candidate Source Provides the most followed / quality followed accounts in a specific country and a geolocation within past 2 weeks. * A "quality follow" refers to any follow that leads to visible engagement, such as favorites, mentions, retweets, direct messages, replies, and quote tweets. The engagement must be allowed in either direction, and must occur on the day of the follow or within one subsequent day. Additionally, there must be no unfollowing, blocking, muting, or reporting of the account in the same time period. * The minimum geolocation precision used is ±20 km (12 mi), and precise user geolocation is not utilized. ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", "strato/config/columns/onboarding:onboarding-strato-client", "strato/config/columns/onboarding/userrecs:userrecs-strato-client", "strato/src/main/scala/com/twitter/strato/client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/PPMILocaleFollowSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSourceParams.CandidateSourceEnabled import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSourceParams.LocaleToExcludeFromRecommendation import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton import com.twitter.strato.generated.client.onboarding.UserPreferredLanguagesOnUserClientColumn import com.twitter.strato.generated.client.onboarding.userrecs.LocaleFollowPpmiClientColumn import com.twitter.timelines.configapi.HasParams /** * Fetches candidates based on the Positive Pointwise Mutual Information (PPMI) statistic * for a set of locales * */ @Singleton class PPMILocaleFollowSource @Inject() ( userPreferredLanguagesOnUserClientColumn: UserPreferredLanguagesOnUserClientColumn, localeFollowPpmiClientColumn: LocaleFollowPpmiClientColumn, statsReceiver: StatsReceiver) extends CandidateSource[HasClientContext with HasParams, CandidateUser] { override val identifier: CandidateSourceIdentifier = PPMILocaleFollowSource.Identifier private val stats = statsReceiver.scope("PPMILocaleFollowSource") override def apply(target: HasClientContext with HasParams): Stitch[Seq[CandidateUser]] = { (for { countryCode <- target.getCountryCode userId <- target.getOptionalUserId } yield { getPreferredLocales(userId, countryCode.toLowerCase()) .flatMap { locale => stats.addGauge("allLocale") { locale.length } val filteredLocale = locale.filter(!target.params(LocaleToExcludeFromRecommendation).contains(_)) stats.addGauge("postFilterLocale") { filteredLocale.length } if (target.params(CandidateSourceEnabled)) { getPPMILocaleFollowCandidates(filteredLocale) } else Stitch(Seq.empty) } .map(_.sortBy(_.score)(Ordering[Option[Double]].reverse) .take(PPMILocaleFollowSource.DefaultMaxCandidatesToReturn)) }).getOrElse(Stitch.Nil) } private def getPPMILocaleFollowCandidates( locales: Seq[String] ): Stitch[Seq[CandidateUser]] = { Stitch .traverse(locales) { locale => // Get PPMI candidates for each locale localeFollowPpmiClientColumn.fetcher .fetch(locale) .map(_.v .map(_.candidates).getOrElse(Nil).map { candidate => CandidateUser(id = candidate.userId, score = Some(candidate.score)) }.map(_.withCandidateSource(identifier))) }.map(_.flatten) } private def getPreferredLocales(userId: Long, countryCode: String): Stitch[Seq[String]] = { userPreferredLanguagesOnUserClientColumn.fetcher .fetch(userId) .map(_.v.map(_.languages).getOrElse(Nil).map { lang => s"$countryCode-$lang".toLowerCase }) } } object PPMILocaleFollowSource { val Identifier = CandidateSourceIdentifier(Algorithm.PPMILocaleFollow.toString) val DefaultMaxCandidatesToReturn = 100 } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/PPMILocaleFollowSourceFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.Param import javax.inject.Inject import javax.inject.Singleton @Singleton class PPMILocaleFollowSourceFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( PPMILocaleFollowSourceParams.CandidateSourceEnabled, ) override val stringSeqFSParams: Seq[Param[Seq[String]] with FSName] = Seq( PPMILocaleFollowSourceParams.LocaleToExcludeFromRecommendation, ) override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( PPMILocaleFollowSourceParams.CandidateSourceWeight, ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/PPMILocaleFollowSourceParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam class PPMILocaleFollowSourceParams {} object PPMILocaleFollowSourceParams { case object LocaleToExcludeFromRecommendation extends FSParam[Seq[String]]( "ppmilocale_follow_source_locales_to_exclude_from_recommendation", default = Seq.empty) case object CandidateSourceEnabled extends FSParam[Boolean]("ppmilocale_follow_source_enabled", true) case object CandidateSourceWeight extends FSBoundedParam[Double]( "ppmilocale_follow_source_candidate_source_weight", default = 1, min = 0.001, max = 2000) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/README.md ================================================ # PPMI Locale Follow Candidate Source Provides accounts based on PPMI ([Positive Pointwise Mutual Information](https://en.wikipedia.org/wiki/Pointwise_mutual_information#Positive_PMI)) using follow actions as a feature for a specific local (language + country) within a week. In simpler terms, it provides a list of the most followed accounts for a given country and language input, based on the PPMI algorithm. PPMI is a statistical measure of the association between two events. In this case, it measures the association between the follow actions and the accounts being followed. In summary, the service utilizes PPMI and follow actions to provide a list of the most followed accounts for a specific country and language input. ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/promoted_accounts/BUILD ================================================ scala_library( platform = "java8", tags = ["bazel-compatible"], dependencies = [ "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "src/thrift/com/twitter/socialgraph:thrift-scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/promoted_accounts/PromotedAccountsCandidateSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.promoted_accounts import com.twitter.adserver.thriftscala.AdServerException import com.twitter.adserver.{thriftscala => adthrift} import com.twitter.finagle.TimeoutException import com.twitter.finagle.stats.Counter import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.clients.adserver.AdRequest import com.twitter.follow_recommendations.common.clients.adserver.AdserverClient import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient import com.twitter.follow_recommendations.common.models.FollowProof import com.twitter.hermit.model.Algorithm import com.twitter.inject.Logging import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton case class PromotedCandidateUser( id: Long, position: Int, adImpression: adthrift.AdImpression, followProof: FollowProof, primaryCandidateSource: Option[CandidateSourceIdentifier]) @Singleton class PromotedAccountsCandidateSource @Inject() ( adserverClient: AdserverClient, sgsClient: SocialGraphClient, statsReceiver: StatsReceiver) extends CandidateSource[AdRequest, PromotedCandidateUser] with Logging { override val identifier: CandidateSourceIdentifier = PromotedAccountsCandidateSource.Identifier val stats: StatsReceiver = statsReceiver.scope(identifier.name) val failureStat: StatsReceiver = stats.scope("failures") val adServerExceptionsCounter: Counter = failureStat.counter("AdServerException") val timeoutCounter: Counter = failureStat.counter("TimeoutException") def apply(request: AdRequest): Stitch[Seq[PromotedCandidateUser]] = { adserverClient .getAdImpressions(request) .rescue { case e: TimeoutException => timeoutCounter.incr() logger.warn("Timeout on Adserver", e) Stitch.Nil case e: AdServerException => adServerExceptionsCounter.incr() logger.warn("Failed to fetch ads", e) Stitch.Nil } .flatMap { adImpressions: Seq[adthrift.AdImpression] => profileNumResults(adImpressions.size, "results_from_ad_server") val idToImpMap = (for { imp <- adImpressions promotedAccountId <- imp.promotedAccountId } yield promotedAccountId -> imp).toMap request.clientContext.userId .map { userId => sgsClient .getIntersections( userId, adImpressions.filter(shouldShowSocialContext).flatMap(_.promotedAccountId), PromotedAccountsCandidateSource.NumIntersections ).map { promotedAccountWithIntersections => idToImpMap.map { case (promotedAccountId, imp) => PromotedCandidateUser( promotedAccountId, imp.insertionPosition .map(_.toInt).getOrElse( getInsertionPositionDefaultValue(request.isTest.getOrElse(false)) ), imp, promotedAccountWithIntersections .getOrElse(promotedAccountId, FollowProof(Nil, 0)), Some(identifier) ) }.toSeq }.onSuccess(result => profileNumResults(result.size, "final_results")) }.getOrElse(Stitch.Nil) } } private def shouldShowSocialContext(imp: adthrift.AdImpression): Boolean = imp.experimentValues.exists { expValues => expValues.get("display.display_style").contains("show_social_context") } private def getInsertionPositionDefaultValue(isTest: Boolean): Int = { if (isTest) 0 else -1 } private def profileNumResults(resultsSize: Int, statName: String): Unit = { if (resultsSize <= 5) { stats.scope(statName).counter(resultsSize.toString).incr() } else { stats.scope(statName).counter("more_than_5").incr() } } } object PromotedAccountsCandidateSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.PromotedAccount.toString) val NumIntersections = 3 } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/promoted_accounts/README.md ================================================ # Promoted Accounts Candidate Source Promoted accounts returned from Ads server. ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "strato/config/columns/onboarding/realGraph:realGraph-strato-client", "strato/config/columns/onboarding/userrecs:userrecs-strato-client", "strato/config/columns/recommendations/twistly:twistly-strato-client", "strato/src/main/scala/com/twitter/strato/client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/README.md ================================================ # RealGraph Candidate Source Provides out-of-network RealGraph candidates for a given user. RealGraph is a user-user graph dataset that aims to measure the strength of the relationship between two users. RealGraph comprises two components: a real-time pipeline that tracks various counts and relationships between user-user edges (such as the number of favorites, replies, retweets, clicks, whether followed, muted, or blocked), and an offline pipeline of a larger set of such user-user edge counts and relationships. Currently, the top k in-network scores have been exported for use by various teams. The RealGraph dataset is used to predict user interactions at Twitter, and is based on the paper "[Realgraph: User interaction prediction at Twitter](http://www.ueo-workshop.com/wp-content/uploads/2014/04/sig-alternate.pdf)" by the UEO workshop at KDD'14. ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/RealGraphOonFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.real_graph import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.Param import javax.inject.Inject import javax.inject.Singleton @Singleton class RealGraphOonFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( RealGraphOonParams.IncludeRealGraphOonCandidates, RealGraphOonParams.TryToReadRealGraphOonCandidates, RealGraphOonParams.UseV2 ) override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( RealGraphOonParams.ScoreThreshold ) override val intFSParams: Seq[FSBoundedParam[Int]] = Seq( RealGraphOonParams.RealGraphOonResultCountThreshold, RealGraphOonParams.MaxResults, ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/RealGraphOonParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.real_graph import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam object RealGraphOonParams { case object IncludeRealGraphOonCandidates extends FSParam[Boolean]( "real_graph_oon_include_candidates", false ) case object TryToReadRealGraphOonCandidates extends FSParam[Boolean]( "real_graph_oon_try_to_read_candidates", false ) case object RealGraphOonResultCountThreshold extends FSBoundedParam[Int]( "real_graph_oon_result_count_threshold", default = 1, min = 0, max = Integer.MAX_VALUE ) case object UseV2 extends FSParam[Boolean]( "real_graph_oon_use_v2", false ) case object ScoreThreshold extends FSBoundedParam[Double]( "real_graph_oon_score_threshold", default = 0.26, min = 0, max = 1.0 ) case object MaxResults extends FSBoundedParam[Int]( "real_graph_oon_max_results", default = 200, min = 0, max = 1000 ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/RealGraphOonV2Source.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.real_graph import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.onboarding.realGraph.UserRealgraphOonV2ClientColumn import com.twitter.timelines.configapi.HasParams import com.twitter.wtf.candidate.thriftscala.CandidateSeq import javax.inject.Inject import javax.inject.Singleton @Singleton class RealGraphOonV2Source @Inject() ( realGraphClientColumn: UserRealgraphOonV2ClientColumn) extends CandidateSource[HasParams with HasClientContext, CandidateUser] { override val identifier: CandidateSourceIdentifier = RealGraphOonV2Source.Identifier override def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] = { request.getOptionalUserId .map { userId => realGraphClientColumn.fetcher .fetch(userId) .map { result => result.v .map { candidates => parseStratoResults(request, candidates) } .getOrElse(Nil) // returned candidates are sorted by score in descending order .take(request.params(RealGraphOonParams.MaxResults)) .map(_.withCandidateSource(identifier)) } }.getOrElse(Stitch(Seq.empty)) } private def parseStratoResults( request: HasParams with HasClientContext, candidateSeqThrift: CandidateSeq ): Seq[CandidateUser] = { candidateSeqThrift.candidates.collect { case candidate if candidate.score >= request.params(RealGraphOonParams.ScoreThreshold) => CandidateUser( candidate.userId, Some(candidate.score) ) } } } object RealGraphOonV2Source { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.RealGraphOonV2.toString ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/RealGraphSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.real_graph import com.twitter.follow_recommendations.common.clients.real_time_real_graph.RealTimeRealGraphClient import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import javax.inject.Inject import javax.inject.Singleton /** * This source gets the already followed edges from the real graph column as a candidate source. */ @Singleton class RealGraphSource @Inject() ( realGraph: RealTimeRealGraphClient) extends CandidateSource[HasParams with HasClientContext, CandidateUser] { override val identifier: CandidateSourceIdentifier = RealGraphSource.Identifier override def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] = { request.getOptionalUserId .map { userId => realGraph.getRealGraphWeights(userId).map { scoreMap => scoreMap.map { case (candidateId, realGraphScore) => CandidateUser(id = candidateId, score = Some(realGraphScore)) .withCandidateSource(identifier) }.toSeq } }.getOrElse(Stitch.Nil) } } object RealGraphSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.RealGraphFollowed.toString) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "discovery-ds/src/main/thrift/com/twitter/dds/jobs/repeated_profile_visits:profile_visit-scala", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "hermit/hermit-core/src/main/scala/com/twitter/hermit/model", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", "src/thrift/com/twitter/experiments/general_metrics:general_metrics-scala", "strato/config/columns/rux:rux-strato-client", "strato/src/main/scala/com/twitter/strato/client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/README.md ================================================ # Recent Engagement Candidate Source Provides recently engaged accounts for a given user: * Explicit engagements: like, retweet, reply * Implicit engagements: profile visit ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RecentEngagementDirectFollowSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.recent_engagement import com.twitter.follow_recommendations.common.clients.real_time_real_graph.RealTimeRealGraphClient import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class RecentEngagementDirectFollowSource @Inject() ( realTimeRealGraphClient: RealTimeRealGraphClient) extends CandidateSource[Long, CandidateUser] { val identifier: CandidateSourceIdentifier = RecentEngagementDirectFollowSource.Identifier /** * Generate a list of candidates for the target using RealtimeGraphClient * and RecentEngagementStore. */ override def apply(targetUserId: Long): Stitch[Seq[CandidateUser]] = { realTimeRealGraphClient .getUsersRecentlyEngagedWith( userId = targetUserId, engagementScoreMap = RealTimeRealGraphClient.EngagementScoreMap, includeDirectFollowCandidates = true, includeNonDirectFollowCandidates = false ) .map(_.map(_.withCandidateSource(identifier)).sortBy(-_.score.getOrElse(0.0))) } } object RecentEngagementDirectFollowSource { val Identifier = CandidateSourceIdentifier(Algorithm.RecentEngagementDirectFollow.toString) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RecentEngagementNonDirectFollowSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.recent_engagement import com.twitter.follow_recommendations.common.clients.real_time_real_graph.RealTimeRealGraphClient import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class RecentEngagementNonDirectFollowSource @Inject() ( realTimeRealGraphClient: RealTimeRealGraphClient) extends CandidateSource[Long, CandidateUser] { val identifier: CandidateSourceIdentifier = RecentEngagementNonDirectFollowSource.Identifier /** * Generate a list of candidates for the target using RealtimeGraphClient * and RecentEngagementStore. */ override def apply(targetUserId: Long): Stitch[Seq[CandidateUser]] = { realTimeRealGraphClient .getUsersRecentlyEngagedWith( userId = targetUserId, engagementScoreMap = RealTimeRealGraphClient.EngagementScoreMap, includeDirectFollowCandidates = false, includeNonDirectFollowCandidates = true ) .map(_.map(_.withCandidateSource(identifier)).sortBy(-_.score.getOrElse(0.0))) } } object RecentEngagementNonDirectFollowSource { val Identifier = CandidateSourceIdentifier(Algorithm.RecentEngagementNonDirectFollow.toString) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RepeatedProfileVisitsFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.recent_engagement import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.Param import javax.inject.Inject import javax.inject.Singleton @Singleton class RepeatedProfileVisitsFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( RepeatedProfileVisitsParams.IncludeCandidates, RepeatedProfileVisitsParams.UseOnlineDataset, ) override val intFSParams: Seq[FSBoundedParam[Int]] = Seq( RepeatedProfileVisitsParams.RecommendationThreshold, RepeatedProfileVisitsParams.BucketingThreshold, ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RepeatedProfileVisitsParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.recent_engagement import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam object RepeatedProfileVisitsParams { // If RepeatedProfileVisitsSource is run and there are recommended candidates for the target user, whether or not // to actually include such candidates in our output recommendations. This FS will be used to control bucketing of // users into control vs treatment buckets. case object IncludeCandidates extends FSParam[Boolean](name = "repeated_profile_visits_include_candidates", default = false) // The threshold at or above which we will consider a profile to have been visited "frequently enough" to recommend // the profile to the target user. case object RecommendationThreshold extends FSBoundedParam[Int]( name = "repeated_profile_visits_recommendation_threshold", default = 3, min = 0, max = Integer.MAX_VALUE) // The threshold at or above which we will consider a profile to have been visited "frequently enough" to recommend // the profile to the target user. case object BucketingThreshold extends FSBoundedParam[Int]( name = "repeated_profile_visits_bucketing_threshold", default = 3, min = 0, max = Integer.MAX_VALUE) // Whether or not to use the online dataset (which has repeated profile visits information updated to within minutes) // instead of the offline dataset (updated via offline jobs, which can have delays of hours to days). case object UseOnlineDataset extends FSParam[Boolean](name = "repeated_profile_visits_use_online_dataset", default = true) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RepeatedProfileVisitsSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.recent_engagement import com.google.inject.Inject import com.google.inject.Singleton import com.twitter.dds.jobs.repeated_profile_visits.thriftscala.ProfileVisitorInfo import com.twitter.experiments.general_metrics.thriftscala.IdType import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.clients.real_time_real_graph.Engagement import com.twitter.follow_recommendations.common.clients.real_time_real_graph.RealTimeRealGraphClient import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.timelines.configapi.HasParams import com.twitter.timelines.configapi.Params import com.twitter.hermit.model.Algorithm import com.twitter.inject.Logging import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.rux.RepeatedProfileVisitsAggregateClientColumn @Singleton class RepeatedProfileVisitsSource @Inject() ( repeatedProfileVisitsAggregateClientColumn: RepeatedProfileVisitsAggregateClientColumn, realTimeRealGraphClient: RealTimeRealGraphClient, statsReceiver: StatsReceiver) extends CandidateSource[HasParams with HasClientContext, CandidateUser] with Logging { val identifier: CandidateSourceIdentifier = RepeatedProfileVisitsSource.Identifier val sourceStatsReceiver = statsReceiver.scope("repeated_profile_visits_source") val offlineFetchErrorCounter = sourceStatsReceiver.counter("offline_fetch_error") val offlineFetchSuccessCounter = sourceStatsReceiver.counter("offline_fetch_success") val onlineFetchErrorCounter = sourceStatsReceiver.counter("online_fetch_error") val onlineFetchSuccessCounter = sourceStatsReceiver.counter("online_fetch_success") val noRepeatedProfileVisitsAboveBucketingThresholdCounter = sourceStatsReceiver.counter("no_repeated_profile_visits_above_bucketing_threshold") val hasRepeatedProfileVisitsAboveBucketingThresholdCounter = sourceStatsReceiver.counter("has_repeated_profile_visits_above_bucketing_threshold") val noRepeatedProfileVisitsAboveRecommendationsThresholdCounter = sourceStatsReceiver.counter("no_repeated_profile_visits_above_recommendations_threshold") val hasRepeatedProfileVisitsAboveRecommendationsThresholdCounter = sourceStatsReceiver.counter("has_repeated_profile_visits_above_recommendations_threshold") val includeCandidatesCounter = sourceStatsReceiver.counter("include_candidates") val noIncludeCandidatesCounter = sourceStatsReceiver.counter("no_include_candidates") // Returns visited user -> visit count, via off dataset. def applyWithOfflineDataset(targetUserId: Long): Stitch[Map[Long, Int]] = { repeatedProfileVisitsAggregateClientColumn.fetcher .fetch(ProfileVisitorInfo(id = targetUserId, idType = IdType.User)).map(_.v) .handle { case e: Throwable => logger.error("Strato fetch for RepeatedProfileVisitsAggregateClientColumn failed: " + e) offlineFetchErrorCounter.incr() None }.onSuccess { result => offlineFetchSuccessCounter.incr() }.map { resultOption => resultOption .flatMap { result => result.profileVisitSet.map { profileVisitSet => profileVisitSet .filter(profileVisit => profileVisit.totalTargetVisitsInLast14Days.getOrElse(0) > 0) .filter(profileVisit => !profileVisit.doesSourceIdFollowTargetId.getOrElse(false)) .flatMap { profileVisit => (profileVisit.targetId, profileVisit.totalTargetVisitsInLast14Days) match { case (Some(targetId), Some(totalVisitsInLast14Days)) => Some(targetId -> totalVisitsInLast14Days) case _ => None } }.toMap[Long, Int] } }.getOrElse(Map.empty) } } // Returns visited user -> visit count, via online dataset. def applyWithOnlineData(targetUserId: Long): Stitch[Map[Long, Int]] = { val visitedUserToEngagementsStitch: Stitch[Map[Long, Seq[Engagement]]] = realTimeRealGraphClient.getRecentProfileViewEngagements(targetUserId) visitedUserToEngagementsStitch .onFailure { f => onlineFetchErrorCounter.incr() }.onSuccess { result => onlineFetchSuccessCounter.incr() }.map { visitedUserToEngagements => visitedUserToEngagements .mapValues(engagements => engagements.size) } } def getRepeatedVisitedAccounts(params: Params, targetUserId: Long): Stitch[Map[Long, Int]] = { var results: Stitch[Map[Long, Int]] = Stitch.value(Map.empty) if (params.getBoolean(RepeatedProfileVisitsParams.UseOnlineDataset)) { results = applyWithOnlineData(targetUserId) } else { results = applyWithOfflineDataset(targetUserId) } // Only keep users that had non-zero engagement counts. results.map(_.filter(input => input._2 > 0)) } def getRecommendations(params: Params, userId: Long): Stitch[Seq[CandidateUser]] = { val recommendationThreshold = params.getInt(RepeatedProfileVisitsParams.RecommendationThreshold) val bucketingThreshold = params.getInt(RepeatedProfileVisitsParams.BucketingThreshold) // Get the list of repeatedly visited profilts. Only keep accounts with >= bucketingThreshold visits. val repeatedVisitedAccountsStitch: Stitch[Map[Long, Int]] = getRepeatedVisitedAccounts(params, userId).map(_.filter(kv => kv._2 >= bucketingThreshold)) repeatedVisitedAccountsStitch.map { candidates => // Now check if we should includeCandidates (e.g. whether user is in control bucket or treatment buckets). if (candidates.isEmpty) { // User has not visited any accounts above bucketing threshold. We will not bucket user into experiment. Just // don't return no candidates. noRepeatedProfileVisitsAboveBucketingThresholdCounter.incr() Seq.empty } else { hasRepeatedProfileVisitsAboveBucketingThresholdCounter.incr() if (!params.getBoolean(RepeatedProfileVisitsParams.IncludeCandidates)) { // User has reached bucketing criteria. We check whether to include candidates (e.g. checking which bucket // the user is in for the experiment). In this case the user is in a bucket to not include any candidates. noIncludeCandidatesCounter.incr() Seq.empty } else { includeCandidatesCounter.incr() // We should include candidates. Include any candidates above recommendation thresholds. val outputCandidatesSeq = candidates .filter(kv => kv._2 >= recommendationThreshold).map { kv => val user = kv._1 val visitCount = kv._2 CandidateUser(user, Some(visitCount.toDouble)) .withCandidateSource(RepeatedProfileVisitsSource.Identifier) }.toSeq if (outputCandidatesSeq.isEmpty) { noRepeatedProfileVisitsAboveRecommendationsThresholdCounter.incr() } else { hasRepeatedProfileVisitsAboveRecommendationsThresholdCounter.incr() } outputCandidatesSeq } } } } override def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] = { request.getOptionalUserId .map { userId => getRecommendations(request.params, userId) }.getOrElse(Stitch.Nil) } } object RepeatedProfileVisitsSource { val Identifier = CandidateSourceIdentifier(Algorithm.RepeatedProfileVisits.toString) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "src/thrift/com/twitter/onboarding/relevance/candidates:candidates-scala", "strato/config/columns/onboarding/userrecs:userrecs-strato-client", "strato/src/main/scala/com/twitter/strato/client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/README.md ================================================ # SALSA Candidate Source Provides an account expansion based on the SALSA PYMK (People You May Know) algorithm for a given account. The algorithm focuses on the mutual follow and address book graph, making it highly effective at providing good mutual follow recommendations. The SALSA algorithm constructs a local graph and performs personalized random walks to identify the best recommendations for the user. The local graph represents the community of users that are most similar to or most relevant to the user, while the personalized random walk identifies the most popular interests among them. For each target user, the local graph is a bipartite graph with a left-hand side (LHS) and a right-hand side (RHS). The LHS is built from several sources, including the target user, forward and reverse address books, mutual follows, recent followings, and recent followers. We choose a specified number of top candidates from these sources for each target user with different weights assigned to each source to favor the corresponding source, and build the LHS using the target user and those top candidates. The RHS consists of two parts: the top candidates from the sources mentioned above for the target user and the mutual follows of the other entries in the LHS. The random walk starts from the target user in the LHS and adopts a restarting strategy to realize personalization. In summary, the SALSA Candidate Source provides an account expansion based on the SALSA PYMK algorithm, utilizing a bipartite graph with personalized random walks to identify the most relevant and interesting recommendations for the user. ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/RecentEngagementDirectFollowSalsaExpansionSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.salsa import com.twitter.follow_recommendations.common.clients.real_time_real_graph.RealTimeRealGraphClient import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class RecentEngagementDirectFollowSalsaExpansionSource @Inject() ( realTimeRealGraphClient: RealTimeRealGraphClient, salsaExpander: SalsaExpander) extends SalsaExpansionBasedCandidateSource[Long](salsaExpander) { override val identifier: CandidateSourceIdentifier = RecentEngagementDirectFollowSalsaExpansionSource.Identifier override def firstDegreeNodes(target: Long): Stitch[Seq[Long]] = realTimeRealGraphClient .getUsersRecentlyEngagedWith( target, RealTimeRealGraphClient.EngagementScoreMap, includeDirectFollowCandidates = true, includeNonDirectFollowCandidates = false ).map { recentlyFollowed => recentlyFollowed .take(RecentEngagementDirectFollowSalsaExpansionSource.NumFirstDegreeNodesToRetrieve) .map(_.id) } override def maxResults(target: Long): Int = RecentEngagementDirectFollowSalsaExpansionSource.OutputSize } object RecentEngagementDirectFollowSalsaExpansionSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.RecentEngagementSarusOcCur.toString) val NumFirstDegreeNodesToRetrieve = 10 val OutputSize = 200 } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/SalsaExpander.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.salsa import com.twitter.finagle.stats.StatsReceiver import com.twitter.strato.generated.client.onboarding.userrecs.SalsaFirstDegreeOnUserClientColumn import com.twitter.strato.generated.client.onboarding.userrecs.SalsaSecondDegreeOnUserClientColumn import com.twitter.follow_recommendations.common.models.AccountProof import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FollowProof import com.twitter.follow_recommendations.common.models.Reason import com.twitter.stitch.Stitch import com.twitter.wtf.candidate.thriftscala.Candidate import javax.inject.Inject import javax.inject.Singleton case class SalsaExpandedCandidate( candidateId: Long, numberOfConnections: Int, totalScore: Double, connectingUsers: Seq[Long]) { def toCandidateUser: CandidateUser = CandidateUser( id = candidateId, score = Some(totalScore), reason = Some(Reason( Some(AccountProof(followProof = Some(FollowProof(connectingUsers, connectingUsers.size)))))) ) } case class SimilarUserCandidate(candidateId: Long, score: Double, similarToCandidate: Long) /** * Salsa expander uses pre-computed lists of candidates for each input user id and returns the highest scored candidates in the pre-computed lists as the expansion for the corresponding input id. */ @Singleton class SalsaExpander @Inject() ( statsReceiver: StatsReceiver, firstDegreeClient: SalsaFirstDegreeOnUserClientColumn, secondDegreeClient: SalsaSecondDegreeOnUserClientColumn, ) { val stats = statsReceiver.scope("salsa_expander") private def similarUsers( input: Seq[Long], neighbors: Seq[Option[Seq[Candidate]]] ): Seq[SalsaExpandedCandidate] = { input .zip(neighbors).flatMap { case (recId, Some(neighbors)) => neighbors.map(neighbor => SimilarUserCandidate(neighbor.userId, neighbor.score, recId)) case _ => Nil }.groupBy(_.candidateId).map { case (key, neighbors) => val scores = neighbors.map(_.score) val connectingUsers = neighbors .sortBy(-_.score) .take(SalsaExpander.MaxConnectingUsersToOutputPerExpandedCandidate) .map(_.similarToCandidate) SalsaExpandedCandidate(key, scores.size, scores.sum, connectingUsers) } .filter( _.numberOfConnections >= math .min(SalsaExpander.MinConnectingUsersThreshold, input.size) ) .toSeq } def apply( firstDegreeInput: Seq[Long], secondDegreeInput: Seq[Long], maxNumOfCandidatesToReturn: Int ): Stitch[Seq[CandidateUser]] = { val firstDegreeNeighborsStitch = Stitch .collect(firstDegreeInput.map(firstDegreeClient.fetcher .fetch(_).map(_.v.map(_.candidates.take(SalsaExpander.MaxDirectNeighbors))))).onSuccess { firstDegreeNeighbors => stats.stat("first_degree_neighbors").add(firstDegreeNeighbors.flatten.size) } val secondDegreeNeighborsStitch = Stitch .collect( secondDegreeInput.map( secondDegreeClient.fetcher .fetch(_).map( _.v.map(_.candidates.take(SalsaExpander.MaxIndirectNeighbors))))).onSuccess { secondDegreeNeighbors => stats.stat("second_degree_neighbors").add(secondDegreeNeighbors.flatten.size) } val neighborStitches = Stitch.join(firstDegreeNeighborsStitch, secondDegreeNeighborsStitch).map { case (first, second) => first ++ second } val similarUsersToInput = neighborStitches.map { neighbors => similarUsers(firstDegreeInput ++ secondDegreeInput, neighbors) } similarUsersToInput.map { // Rank the candidate cot users by the combined weights from the connecting users. This is the default original implementation. It is unlikely to have weight ties and thus a second ranking function is not necessary. _.sortBy(-_.totalScore) .take(maxNumOfCandidatesToReturn) .map(_.toCandidateUser) } } } object SalsaExpander { val MaxDirectNeighbors = 2000 val MaxIndirectNeighbors = 2000 val MinConnectingUsersThreshold = 2 val MaxConnectingUsersToOutputPerExpandedCandidate = 3 } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/SalsaExpansionBasedCandidateSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.salsa import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.stitch.Stitch abstract class SalsaExpansionBasedCandidateSource[Target](salsaExpander: SalsaExpander) extends CandidateSource[Target, CandidateUser] { // Define first/second degree as empty sequences in cases of subclasses // that don't implement one or the other. // Example: MagicRecs only uses first degree nodes, and can ignore implementing secondDegreeNodes // // This allows apply(target) to combine both in the base class def firstDegreeNodes(target: Target): Stitch[Seq[Long]] = Stitch.value(Seq()) def secondDegreeNodes(target: Target): Stitch[Seq[Long]] = Stitch.value(Seq()) // max number output results def maxResults(target: Target): Int override def apply(target: Target): Stitch[Seq[CandidateUser]] = { val nodes = Stitch.join(firstDegreeNodes(target), secondDegreeNodes(target)) nodes.flatMap { case (firstDegreeCandidates, secondDegreeCandidates) => { salsaExpander(firstDegreeCandidates, secondDegreeCandidates, maxResults(target)) .map(_.map(_.withCandidateSource(identifier)).sortBy(-_.score.getOrElse(0.0))) } } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", "strato/config/columns/onboarding/userrecs:userrecs-strato-client", "strato/config/columns/recommendations/follow2vec:follow2vec-strato-client", "strato/config/columns/recommendations/similarity:similarity-strato-client", "strato/src/main/scala/com/twitter/strato/client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/CacheBasedSimsStore.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims import com.twitter.escherbird.util.stitchcache.StitchCache import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasSimilarToContext import com.twitter.hermit.candidate.thriftscala.Candidates import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.timelines.configapi.HasParams import com.twitter.util.Duration import java.lang.{Long => JLong} class CacheBasedSimsStore( id: CandidateSourceIdentifier, fetcher: Fetcher[Long, Unit, Candidates], maxCacheSize: Int, cacheTtl: Duration, statsReceiver: StatsReceiver) extends CandidateSource[HasParams with HasSimilarToContext, CandidateUser] { override val identifier: CandidateSourceIdentifier = id private def getUsersFromSimsSource(userId: JLong): Stitch[Option[Candidates]] = { fetcher .fetch(userId) .map(_.v) } private val simsCache = StitchCache[JLong, Option[Candidates]]( maxCacheSize = maxCacheSize, ttl = cacheTtl, statsReceiver = statsReceiver, underlyingCall = getUsersFromSimsSource ) override def apply(request: HasParams with HasSimilarToContext): Stitch[Seq[CandidateUser]] = { Stitch .traverse(request.similarToUserIds) { userId => simsCache.readThrough(userId).map { candidatesOpt => candidatesOpt .map { candidates => StratoBasedSimsCandidateSource.map(userId, candidates) }.getOrElse(Nil) } }.map(_.flatten.distinct.map(_.withCandidateSource(identifier))) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/DBV2SimsRefreshStore.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims import com.google.inject.Singleton import com.twitter.finagle.stats.StatsReceiver import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.strato.generated.client.onboarding.userrecs.NewSimsRefreshOnUserClientColumn import com.twitter.util.Duration import javax.inject.Inject @Singleton class DBV2SimsRefreshStore @Inject() ( newSimsRefreshOnUserClientColumn: NewSimsRefreshOnUserClientColumn) extends StratoBasedSimsCandidateSourceWithUnitView( fetcher = newSimsRefreshOnUserClientColumn.fetcher, identifier = DBV2SimsRefreshStore.Identifier) @Singleton class CachedDBV2SimsRefreshStore @Inject() ( newSimsRefreshOnUserClientColumn: NewSimsRefreshOnUserClientColumn, statsReceiver: StatsReceiver) extends CacheBasedSimsStore( id = DBV2SimsRefreshStore.Identifier, fetcher = newSimsRefreshOnUserClientColumn.fetcher, maxCacheSize = DBV2SimsRefreshStore.MaxCacheSize, cacheTtl = DBV2SimsRefreshStore.CacheTTL, statsReceiver = statsReceiver.scope("CachedDBV2SimsRefreshStore", "cache") ) object DBV2SimsRefreshStore { val Identifier = CandidateSourceIdentifier(Algorithm.Sims.toString) val MaxCacheSize = 5000 val CacheTTL: Duration = Duration.fromHours(24) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/DBV2SimsStore.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims import com.google.inject.Singleton import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.hermit.candidate.thriftscala.Candidates import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.strato.client.Fetcher import com.twitter.util.Duration import javax.inject.Inject @Singleton class DBV2SimsStore @Inject() ( @Named(GuiceNamedConstants.DBV2_SIMS_FETCHER) fetcher: Fetcher[Long, Unit, Candidates]) extends StratoBasedSimsCandidateSourceWithUnitView( fetcher, identifier = DBV2SimsStore.Identifier) @Singleton class CachedDBV2SimsStore @Inject() ( @Named(GuiceNamedConstants.DBV2_SIMS_FETCHER) fetcher: Fetcher[Long, Unit, Candidates], statsReceiver: StatsReceiver) extends CacheBasedSimsStore( id = DBV2SimsStore.Identifier, fetcher = fetcher, maxCacheSize = DBV2SimsStore.MaxCacheSize, cacheTtl = DBV2SimsStore.CacheTTL, statsReceiver = statsReceiver.scope("CachedDBV2SimsStore", "cache") ) object DBV2SimsStore { val Identifier = CandidateSourceIdentifier(Algorithm.Sims.toString) val MaxCacheSize = 1000 val CacheTTL: Duration = Duration.fromHours(24) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/Follow2vecNearestNeighborsStore.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims import com.google.inject.Singleton import com.twitter.follow_recommendations.common.candidate_sources.sims.Follow2vecNearestNeighborsStore.NearestNeighborParamsType import com.twitter.hermit.candidate.thriftscala.Candidate import com.twitter.hermit.candidate.thriftscala.Candidates import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import com.twitter.strato.catalog.Fetch import com.twitter.strato.client.Fetcher import com.twitter.strato.generated.client.recommendations.follow2vec.LinearRegressionFollow2vecNearestNeighborsClientColumn import com.twitter.util.Return import com.twitter.util.Throw import javax.inject.Inject @Singleton class LinearRegressionFollow2vecNearestNeighborsStore @Inject() ( linearRegressionFollow2vecNearestNeighborsClientColumn: LinearRegressionFollow2vecNearestNeighborsClientColumn) extends StratoBasedSimsCandidateSource[NearestNeighborParamsType]( Follow2vecNearestNeighborsStore.convertFetcher( linearRegressionFollow2vecNearestNeighborsClientColumn.fetcher), view = Follow2vecNearestNeighborsStore.defaultSearchParams, identifier = Follow2vecNearestNeighborsStore.IdentifierF2vLinearRegression ) object Follow2vecNearestNeighborsStore { // (userid, feature store version for data) type NearestNeighborKeyType = (Long, Long) // (neighbors to be returned, ef value: accuracy / latency tradeoff, distance for filtering) type NearestNeighborParamsType = (Option[Int], Option[Int], Option[Double]) // (seq(found neighbor id, score), distance for filtering) type NearestNeighborValueType = (Seq[(Long, Option[Double])], Option[Double]) val IdentifierF2vLinearRegression: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.LinearRegressionFollow2VecNearestNeighbors.toString) val defaultFeatureStoreVersion: Long = 20210708 val defaultSearchParams: NearestNeighborParamsType = (None, None, None) def convertFetcher( fetcher: Fetcher[NearestNeighborKeyType, NearestNeighborParamsType, NearestNeighborValueType] ): Fetcher[Long, NearestNeighborParamsType, Candidates] = { (key: Long, view: NearestNeighborParamsType) => { def toCandidates( results: Option[NearestNeighborValueType] ): Option[Candidates] = { results.flatMap { r => Some( Candidates( key, r._1.map { neighbor => Candidate(neighbor._1, neighbor._2.getOrElse(0)) } ) ) } } val results: Stitch[Fetch.Result[NearestNeighborValueType]] = fetcher.fetch(key = (key, defaultFeatureStoreVersion), view = view) results.transform { case Return(r) => Stitch.value(Fetch.Result(toCandidates(r.v))) case Throw(e) => Stitch.exception(e) } } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/README.md ================================================ # Sims Candidate Source Offers various online sources for finding similar accounts based on a given user, whether it is the target user or an account candidate. ## Sims The objective is to identify a list of K users who are similar to a given user. In this scenario, we primarily focus on finding similar users as "producers" rather than "consumers." Sims has two steps: candidate generation and ranking. ### Sims Candidate Generation With over 700 million users to consider, there are multiple ways to define similarities. Currently, we have three candidate sources for Sims: **CosineFollow** (based on user-user follow graph): The similarity between two users is defined as the cosine similarity between their followers. Despite sounding simple, computing all-pair similarity on the entire follow graph is computationally challenging. We are currently using the WHIMP algorithm to find the top 1000 similar users for each user ID. This candidate source has the largest coverage, as it can find similar user candidates for more than 700 million users. **CosineList** (based on user-list membership graph): The similarity between two users is defined as the cosine similarity between the lists they are included as members (e.g., [here](https://twitter.com/jack/lists/memberships) are the lists that @jack is on). The same algorithm as CosineFollow is used. **Follow2Vec** (essentially Word2Vec on user-user follow graph): We first train the Word2Vec model on follow sequence data to obtain users' embeddings and then find the most similar users based on the similarity of the embeddings. However, we need enough data for each user to learn a meaningful embedding for them, so we can only obtain embeddings for the top 10 million users (currently in production, testing 30 million users). Furthermore, Word2Vec model training is limited by memory and computation as it is trained on a single machine. ##### Cosine Similarity A crucial component in Sims is calculating cosine similarities between users based on a user-X (X can be a user, list, or other entities) bipartite graph. This problem is technically challenging and took several years of effort to solve. The current implementation uses the algorithm proposed in [When hashes met wedges: A distributed algorithm for finding high similarity vectors. WWW 2017](https://arxiv.org/pdf/1703.01054.pdf) ### Sims Ranking After the candidate generation step, we can obtain dozens to hundreds of similar user candidates for each user. However, since these candidates come from different algorithms, we need a way to rank them. To do this, we collect user feedback. We use the "Profile Sidebar Impressions & Follow" (a module with follow suggestions displayed when a user visits a profile page and scrolls down) to collect training data. To alleviate any system bias, we use 4% of traffic to show randomly shuffled candidates to users and collect positive (followed impression) and negative (impression only) data from this traffic. This data is used as an evaluation set. We use a portion of the remaining 96% of traffic for training data, filtering only for sets of impressions that had at least one follow, ensuring that the user taking action was paying attention to the impressions. The examples are in the format of (profile_user, candidate_user, label). We add features for profile_users and candidate_users based on some high-level aggregated statistics in a feature dataset provided by the Customer Journey team, as well as features that represent the similarity between the profile_user and candidate_user. We employ a multi-tower MLP model and optimize the logistic loss. The model is refreshed weekly using an ML workflow. We recompute the candidates and rank them daily. The ranked results are published to the Manhattan dataset. ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SimsExperimentalStore.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims import com.google.inject.Singleton import com.twitter.finagle.stats.StatsReceiver import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.strato.generated.client.recommendations.similarity.SimilarUsersBySimsExperimentalOnUserClientColumn import com.twitter.util.Duration import javax.inject.Inject @Singleton class SimsExperimentalStore @Inject() ( simsExperimentalOnUserClientColumn: SimilarUsersBySimsExperimentalOnUserClientColumn) extends StratoBasedSimsCandidateSourceWithUnitView( fetcher = simsExperimentalOnUserClientColumn.fetcher, identifier = SimsExperimentalStore.Identifier ) @Singleton class CachedSimsExperimentalStore @Inject() ( simsExperimentalOnUserClientColumn: SimilarUsersBySimsExperimentalOnUserClientColumn, statsReceiver: StatsReceiver) extends CacheBasedSimsStore( id = SimsExperimentalStore.Identifier, fetcher = simsExperimentalOnUserClientColumn.fetcher, maxCacheSize = SimsExperimentalStore.MaxCacheSize, cacheTtl = SimsExperimentalStore.CacheTTL, statsReceiver = statsReceiver.scope("CachedSimsExperimentalStore", "cache") ) object SimsExperimentalStore { val Identifier = CandidateSourceIdentifier(Algorithm.Sims.toString) val MaxCacheSize = 1000 val CacheTTL: Duration = Duration.fromHours(12) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SimsSourceFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @Singleton class SimsSourceFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[FSParam[Boolean] with FSName] = Seq( SimsSourceParams.DisableHeavyRanker ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SimsSourceParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims import com.twitter.timelines.configapi.FSParam object SimsSourceParams { case object EnableDBV2SimsStore extends FSParam[Boolean]("sims_source_enable_dbv2_source", false) case object EnableDBV2SimsRefreshStore extends FSParam[Boolean]("sims_source_enable_dbv2_refresh_source", false) case object EnableExperimentalSimsStore extends FSParam[Boolean]("sims_source_enable_experimental_source", false) case object DisableHeavyRanker extends FSParam[Boolean]("sims_source_disable_heavy_ranker", default = false) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SimsStore.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims import com.google.inject.Singleton import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.hermit.candidate.thriftscala.Candidates import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.strato.client.Fetcher import com.twitter.util.Duration import javax.inject.Inject @Singleton class SimsStore @Inject() ( @Named(GuiceNamedConstants.SIMS_FETCHER) fetcher: Fetcher[Long, Unit, Candidates]) extends StratoBasedSimsCandidateSourceWithUnitView(fetcher, identifier = SimsStore.Identifier) @Singleton class CachedSimsStore @Inject() ( @Named(GuiceNamedConstants.SIMS_FETCHER) fetcher: Fetcher[Long, Unit, Candidates], statsReceiver: StatsReceiver) extends CacheBasedSimsStore( id = SimsStore.Identifier, fetcher = fetcher, maxCacheSize = SimsStore.MaxCacheSize, cacheTtl = SimsStore.CacheTTL, statsReceiver = statsReceiver.scope("CachedSimsStore", "cache") ) object SimsStore { val Identifier = CandidateSourceIdentifier(Algorithm.Sims.toString) val MaxCacheSize = 50000 val CacheTTL: Duration = Duration.fromHours(24) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/StratoBasedSimsCandidateSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims import com.twitter.follow_recommendations.common.candidate_sources.base.StratoFetcherSource import com.twitter.follow_recommendations.common.models.AccountProof import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.Reason import com.twitter.follow_recommendations.common.models.SimilarToProof import com.twitter.hermit.candidate.thriftscala.Candidates import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.strato.client.Fetcher abstract class StratoBasedSimsCandidateSource[U]( fetcher: Fetcher[Long, U, Candidates], view: U, override val identifier: CandidateSourceIdentifier) extends StratoFetcherSource[Long, U, Candidates](fetcher, view, identifier) { override def map(target: Long, candidates: Candidates): Seq[CandidateUser] = StratoBasedSimsCandidateSource.map(target, candidates) } object StratoBasedSimsCandidateSource { def map(target: Long, candidates: Candidates): Seq[CandidateUser] = { for { candidate <- candidates.candidates } yield CandidateUser( id = candidate.userId, score = Some(candidate.score), reason = Some( Reason( Some( AccountProof( similarToProof = Some(SimilarToProof(Seq(target))) ) ) ) ) ) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/StratoBasedSimsCandidateSourceWithUnitView.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims import com.twitter.hermit.candidate.thriftscala.Candidates import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.strato.client.Fetcher abstract class StratoBasedSimsCandidateSourceWithUnitView( fetcher: Fetcher[Long, Unit, Candidates], override val identifier: CandidateSourceIdentifier) extends StratoBasedSimsCandidateSource[Unit](fetcher, Unit, identifier) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SwitchingSimsSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasSimilarToContext import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import javax.inject.Inject import javax.inject.Singleton @Singleton class SwitchingSimsSource @Inject() ( cachedDBV2SimsStore: CachedDBV2SimsStore, cachedDBV2SimsRefreshStore: CachedDBV2SimsRefreshStore, cachedSimsExperimentalStore: CachedSimsExperimentalStore, cachedSimsStore: CachedSimsStore, statsReceiver: StatsReceiver = NullStatsReceiver) extends CandidateSource[HasParams with HasSimilarToContext, CandidateUser] { override val identifier: CandidateSourceIdentifier = SwitchingSimsSource.Identifier private val stats = statsReceiver.scope("SwitchingSimsSource") private val dbV2SimsStoreCounter = stats.counter("DBV2SimsStore") private val dbV2SimsRefreshStoreCounter = stats.counter("DBV2SimsRefreshStore") private val simsExperimentalStoreCounter = stats.counter("SimsExperimentalStore") private val simsStoreCounter = stats.counter("SimsStore") override def apply(request: HasParams with HasSimilarToContext): Stitch[Seq[CandidateUser]] = { val selectedSimsStore = if (request.params(SimsSourceParams.EnableDBV2SimsStore)) { dbV2SimsStoreCounter.incr() cachedDBV2SimsStore } else if (request.params(SimsSourceParams.EnableDBV2SimsRefreshStore)) { dbV2SimsRefreshStoreCounter.incr() cachedDBV2SimsRefreshStore } else if (request.params(SimsSourceParams.EnableExperimentalSimsStore)) { simsExperimentalStoreCounter.incr() cachedSimsExperimentalStore } else { simsStoreCounter.incr() cachedSimsStore } stats.counter("total").incr() selectedSimsStore(request) } } object SwitchingSimsSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier(Algorithm.Sims.toString) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "strato/src/main/scala/com/twitter/strato/client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/DBV2SimsExpansionParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam object DBV2SimsExpansionParams { // Theses divisors are used to calibrate DBv2Sims extension candidates scores case object RecentFollowingSimilarUsersDBV2CalibrateDivisor extends FSBoundedParam[Double]( "sims_expansion_recent_following_similar_users_dbv2_divisor", default = 1.0d, min = 0.1d, max = 100d) case object RecentEngagementSimilarUsersDBV2CalibrateDivisor extends FSBoundedParam[Double]( "sims_expansion_recent_engagement_similar_users_dbv2_divisor", default = 1.0d, min = 0.1d, max = 100d) case object DisableHeavyRanker extends FSParam[Boolean]("sims_expansion_disable_heavy_ranker", default = false) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/README.md ================================================ # Sims Expansion Candidate Source provides similar accounts based on the Sims algorithm for a given set of accounts. This is a 2nd-hop expansion, meaning that the input accounts could be a user's recently engaged, followed, or algorithm-generated (such as RealGraph) accounts. For more information on Sims and how it is utilized in the Follow Recommendations Service, please refer to the `follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/README.md` file. ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentEngagementSimilarUsersFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @Singleton class RecentEngagementSimilarUsersFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[FSParam[Boolean]] = Seq( RecentEngagementSimilarUsersParams.FirstDegreeSortEnabled ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentEngagementSimilarUsersParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSParam object RecentEngagementSimilarUsersParams { case object FirstDegreeSortEnabled extends FSParam[Boolean]( name = "sims_expansion_recent_engagement_first_degree_sort", default = true) case object Aggregator extends FSEnumParam[SimsExpansionSourceAggregatorId.type]( name = "sims_expansion_recent_engagement_aggregator_id", default = SimsExpansionSourceAggregatorId.Sum, enum = SimsExpansionSourceAggregatorId) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentEngagementSimilarUsersSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.candidate_sources.sims.SwitchingSimsSource import com.twitter.follow_recommendations.common.clients.real_time_real_graph.RealTimeRealGraphClient import com.twitter.follow_recommendations.common.models.AccountProof import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.Reason import com.twitter.follow_recommendations.common.models.SimilarToProof import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import javax.inject.Inject import javax.inject.Singleton @Singleton class RecentEngagementSimilarUsersSource @Inject() ( realTimeRealGraphClient: RealTimeRealGraphClient, switchingSimsSource: SwitchingSimsSource, statsReceiver: StatsReceiver) extends SimsExpansionBasedCandidateSource[HasClientContext with HasParams]( switchingSimsSource) { override def maxSecondaryDegreeNodes(req: HasClientContext with HasParams): Int = Int.MaxValue override def maxResults(req: HasClientContext with HasParams): Int = RecentEngagementSimilarUsersSource.MaxResults override val identifier: CandidateSourceIdentifier = RecentEngagementSimilarUsersSource.Identifier private val stats = statsReceiver.scope(identifier.name) private val calibratedScoreCounter = stats.counter("calibrated_scores_counter") override def scoreCandidate(sourceScore: Double, similarToScore: Double): Double = { sourceScore * similarToScore } override def calibrateDivisor(req: HasClientContext with HasParams): Double = { req.params(DBV2SimsExpansionParams.RecentEngagementSimilarUsersDBV2CalibrateDivisor) } override def calibrateScore( candidateScore: Double, req: HasClientContext with HasParams ): Double = { calibratedScoreCounter.incr() candidateScore / calibrateDivisor(req) } /** * fetch first degree nodes given request */ override def firstDegreeNodes( target: HasClientContext with HasParams ): Stitch[Seq[CandidateUser]] = { target.getOptionalUserId .map { userId => realTimeRealGraphClient .getUsersRecentlyEngagedWith( userId, RealTimeRealGraphClient.EngagementScoreMap, includeDirectFollowCandidates = true, includeNonDirectFollowCandidates = true ).map(_.sortBy(-_.score.getOrElse(0.0d)) .take(RecentEngagementSimilarUsersSource.MaxFirstDegreeNodes)) }.getOrElse(Stitch.Nil) } override def aggregateAndScore( request: HasClientContext with HasParams, firstDegreeToSecondDegreeNodesMap: Map[CandidateUser, Seq[SimilarUser]] ): Stitch[Seq[CandidateUser]] = { val inputNodes = firstDegreeToSecondDegreeNodesMap.keys.map(_.id).toSet val aggregator = request.params(RecentEngagementSimilarUsersParams.Aggregator) match { case SimsExpansionSourceAggregatorId.Max => SimsExpansionBasedCandidateSource.ScoreAggregator.Max case SimsExpansionSourceAggregatorId.Sum => SimsExpansionBasedCandidateSource.ScoreAggregator.Sum case SimsExpansionSourceAggregatorId.MultiDecay => SimsExpansionBasedCandidateSource.ScoreAggregator.MultiDecay } val groupedCandidates = firstDegreeToSecondDegreeNodesMap.values.flatten .filterNot(c => inputNodes.contains(c.candidateId)) .groupBy(_.candidateId) .map { case (id, candidates) => // Different aggregators for final score val finalScore = aggregator(candidates.map(_.score).toSeq) val proofs = candidates.map(_.similarTo).toSet CandidateUser( id = id, score = Some(finalScore), reason = Some(Reason(Some(AccountProof(similarToProof = Some(SimilarToProof(proofs.toSeq)))))) ).withCandidateSource(identifier) } .toSeq .sortBy(-_.score.getOrElse(0.0d)) .take(maxResults(request)) Stitch.value(groupedCandidates) } } object RecentEngagementSimilarUsersSource { val Identifier = CandidateSourceIdentifier(Algorithm.RecentEngagementSimilarUser.toString) val MaxFirstDegreeNodes = 10 val MaxResults = 200 } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentFollowingSimilarUsersParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam object RecentFollowingSimilarUsersParams { case object MaxFirstDegreeNodes extends FSBoundedParam[Int]( name = "sims_expansion_recent_following_max_first_degree_nodes", default = 10, min = 0, max = 200) case object MaxSecondaryDegreeExpansionPerNode extends FSBoundedParam[Int]( name = "sims_expansion_recent_following_max_secondary_degree_nodes", default = 40, min = 0, max = 200) case object MaxResults extends FSBoundedParam[Int]( name = "sims_expansion_recent_following_max_results", default = 200, min = 0, max = 200) case object TimestampIntegrated extends FSParam[Boolean]( name = "sims_expansion_recent_following_integ_timestamp", default = false) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentFollowingSimilarUsersSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion import com.google.inject.Singleton import com.twitter.follow_recommendations.common.candidate_sources.sims.SwitchingSimsSource import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import javax.inject.Inject object RecentFollowingSimilarUsersSource { val Identifier = CandidateSourceIdentifier(Algorithm.NewFollowingSimilarUser.toString) } @Singleton class RecentFollowingSimilarUsersSource @Inject() ( socialGraph: SocialGraphClient, switchingSimsSource: SwitchingSimsSource, statsReceiver: StatsReceiver) extends SimsExpansionBasedCandidateSource[ HasParams with HasRecentFollowedUserIds with HasClientContext ](switchingSimsSource) { val identifier = RecentFollowingSimilarUsersSource.Identifier private val stats = statsReceiver.scope(identifier.name) private val maxResultsStats = stats.scope("max_results") private val calibratedScoreCounter = stats.counter("calibrated_scores_counter") override def firstDegreeNodes( request: HasParams with HasRecentFollowedUserIds with HasClientContext ): Stitch[Seq[CandidateUser]] = { if (request.params(RecentFollowingSimilarUsersParams.TimestampIntegrated)) { val recentFollowedUserIdsWithTimeStitch = socialGraph.getRecentFollowedUserIdsWithTime(request.clientContext.userId.get) recentFollowedUserIdsWithTimeStitch.map { results => val first_degree_nodes = results .sortBy(-_.timeInMs).take( request.params(RecentFollowingSimilarUsersParams.MaxFirstDegreeNodes)) val max_timestamp = first_degree_nodes.head.timeInMs first_degree_nodes.map { case userIdWithTime => CandidateUser( userIdWithTime.userId, score = Some(userIdWithTime.timeInMs.toDouble / max_timestamp)) } } } else { Stitch.value( request.recentFollowedUserIds .getOrElse(Nil).take( request.params(RecentFollowingSimilarUsersParams.MaxFirstDegreeNodes)).map( CandidateUser(_, score = Some(1.0))) ) } } override def maxSecondaryDegreeNodes( req: HasParams with HasRecentFollowedUserIds with HasClientContext ): Int = { req.params(RecentFollowingSimilarUsersParams.MaxSecondaryDegreeExpansionPerNode) } override def maxResults( req: HasParams with HasRecentFollowedUserIds with HasClientContext ): Int = { val firstDegreeNodes = req.params(RecentFollowingSimilarUsersParams.MaxFirstDegreeNodes) val maxResultsNum = req.params(RecentFollowingSimilarUsersParams.MaxResults) maxResultsStats .stat( s"RecentFollowingSimilarUsersSource_firstDegreeNodes_${firstDegreeNodes}_maxResults_${maxResultsNum}") .add(1) maxResultsNum } override def scoreCandidate(sourceScore: Double, similarToScore: Double): Double = { sourceScore * similarToScore } override def calibrateDivisor( req: HasParams with HasRecentFollowedUserIds with HasClientContext ): Double = { req.params(DBV2SimsExpansionParams.RecentFollowingSimilarUsersDBV2CalibrateDivisor) } override def calibrateScore( candidateScore: Double, req: HasParams with HasRecentFollowedUserIds with HasClientContext ): Double = { calibratedScoreCounter.incr() candidateScore / calibrateDivisor(req) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentStrongEngagementDirectFollowSimilarUsersSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion import com.google.inject.Singleton import com.twitter.follow_recommendations.common.candidate_sources.sims.SwitchingSimsSource import com.twitter.follow_recommendations.common.clients.real_time_real_graph.RealTimeRealGraphClient import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import javax.inject.Inject @Singleton class RecentStrongEngagementDirectFollowSimilarUsersSource @Inject() ( realTimeRealGraphClient: RealTimeRealGraphClient, switchingSimsSource: SwitchingSimsSource) extends SimsExpansionBasedCandidateSource[HasClientContext with HasParams]( switchingSimsSource) { val identifier = RecentStrongEngagementDirectFollowSimilarUsersSource.Identifier override def firstDegreeNodes( request: HasClientContext with HasParams ): Stitch[Seq[CandidateUser]] = request.getOptionalUserId .map { userId => realTimeRealGraphClient .getUsersRecentlyEngagedWith( userId, RealTimeRealGraphClient.StrongEngagementScoreMap, includeDirectFollowCandidates = true, includeNonDirectFollowCandidates = false ).map(_.take(RecentStrongEngagementDirectFollowSimilarUsersSource.MaxFirstDegreeNodes)) }.getOrElse(Stitch.Nil) override def maxSecondaryDegreeNodes(request: HasClientContext with HasParams): Int = Int.MaxValue override def maxResults(request: HasClientContext with HasParams): Int = RecentStrongEngagementDirectFollowSimilarUsersSource.MaxResults override def scoreCandidate(sourceScore: Double, similarToScore: Double): Double = { sourceScore * similarToScore } override def calibrateDivisor(req: HasClientContext with HasParams): Double = 1.0d } object RecentStrongEngagementDirectFollowSimilarUsersSource { val Identifier = CandidateSourceIdentifier(Algorithm.RecentStrongEngagementSimilarUser.toString) val MaxFirstDegreeNodes = 10 val MaxResults = 200 } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/SimsExpansionBasedCandidateSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion import com.twitter.follow_recommendations.common.candidate_sources.base.TwoHopExpansionCandidateSource import com.twitter.follow_recommendations.common.candidate_sources.sims.SwitchingSimsSource import com.twitter.follow_recommendations.common.models.AccountProof import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasSimilarToContext import com.twitter.follow_recommendations.common.models.Reason import com.twitter.follow_recommendations.common.models.SimilarToProof import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import scala.math._ case class SimilarUser(candidateId: Long, similarTo: Long, score: Double) abstract class SimsExpansionBasedCandidateSource[-Target <: HasParams]( switchingSimsSource: SwitchingSimsSource) extends TwoHopExpansionCandidateSource[Target, CandidateUser, SimilarUser, CandidateUser] { // max number secondary degree nodes per first degree node def maxSecondaryDegreeNodes(req: Target): Int // max number output results def maxResults(req: Target): Int // scorer to score candidate based on first and second degree node scores def scoreCandidate(source: Double, similarToScore: Double): Double def calibrateDivisor(req: Target): Double def calibrateScore(candidateScore: Double, req: Target): Double = { candidateScore / calibrateDivisor(req) } override def secondaryDegreeNodes(req: Target, node: CandidateUser): Stitch[Seq[SimilarUser]] = { switchingSimsSource(new HasParams with HasSimilarToContext { override val similarToUserIds = Seq(node.id) override val params = (req.params) }).map(_.take(maxSecondaryDegreeNodes(req)).map { candidate => SimilarUser( candidate.id, node.id, (node.score, candidate.score) match { // only calibrated sims expanded candidates scores case (Some(nodeScore), Some(candidateScore)) => calibrateScore(scoreCandidate(nodeScore, candidateScore), req) case (Some(nodeScore), _) => nodeScore // NewFollowingSimilarUser will enter this case case _ => calibrateScore(candidate.score.getOrElse(0.0), req) } ) }) } override def aggregateAndScore( request: Target, firstDegreeToSecondDegreeNodesMap: Map[CandidateUser, Seq[SimilarUser]] ): Stitch[Seq[CandidateUser]] = { val inputNodes = firstDegreeToSecondDegreeNodesMap.keys.map(_.id).toSet val aggregator = request.params(SimsExpansionSourceParams.Aggregator) match { case SimsExpansionSourceAggregatorId.Max => SimsExpansionBasedCandidateSource.ScoreAggregator.Max case SimsExpansionSourceAggregatorId.Sum => SimsExpansionBasedCandidateSource.ScoreAggregator.Sum case SimsExpansionSourceAggregatorId.MultiDecay => SimsExpansionBasedCandidateSource.ScoreAggregator.MultiDecay } val groupedCandidates = firstDegreeToSecondDegreeNodesMap.values.flatten .filterNot(c => inputNodes.contains(c.candidateId)) .groupBy(_.candidateId) .map { case (id, candidates) => // Different aggregators for final score val finalScore = aggregator(candidates.map(_.score).toSeq) val proofs = candidates.map(_.similarTo).toSet CandidateUser( id = id, score = Some(finalScore), reason = Some(Reason(Some(AccountProof(similarToProof = Some(SimilarToProof(proofs.toSeq)))))) ).withCandidateSource(identifier) } .toSeq .sortBy(-_.score.getOrElse(0.0d)) .take(maxResults(request)) Stitch.value(groupedCandidates) } } object SimsExpansionBasedCandidateSource { object ScoreAggregator { val Max: Seq[Double] => Double = (candidateScores: Seq[Double]) => { if (candidateScores.size > 0) candidateScores.max else 0.0 } val Sum: Seq[Double] => Double = (candidateScores: Seq[Double]) => { candidateScores.sum } val MultiDecay: Seq[Double] => Double = (candidateScores: Seq[Double]) => { val alpha = 0.1 val beta = 0.1 val gamma = 0.8 val decay_scores: Seq[Double] = candidateScores .sorted(Ordering[Double].reverse) .zipWithIndex .map(x => x._1 * pow(gamma, x._2)) alpha * candidateScores.max + decay_scores.sum + beta * candidateScores.size } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/SimsExpansionFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @Singleton class SimsExpansionFSConfig @Inject() () extends FeatureSwitchConfig { override val intFSParams: Seq[FSBoundedParam[Int]] = Seq( RecentFollowingSimilarUsersParams.MaxFirstDegreeNodes, RecentFollowingSimilarUsersParams.MaxSecondaryDegreeExpansionPerNode, RecentFollowingSimilarUsersParams.MaxResults ) override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( DBV2SimsExpansionParams.RecentFollowingSimilarUsersDBV2CalibrateDivisor, DBV2SimsExpansionParams.RecentEngagementSimilarUsersDBV2CalibrateDivisor ) override val booleanFSParams: Seq[FSParam[Boolean]] = Seq( DBV2SimsExpansionParams.DisableHeavyRanker, RecentFollowingSimilarUsersParams.TimestampIntegrated ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/SimsExpansionSourceParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion import com.twitter.timelines.configapi.FSEnumParam object SimsExpansionSourceParams { case object Aggregator extends FSEnumParam[SimsExpansionSourceAggregatorId.type]( name = "sims_expansion_aggregator_id", default = SimsExpansionSourceAggregatorId.Sum, enum = SimsExpansionSourceAggregatorId) } object SimsExpansionSourceAggregatorId extends Enumeration { type AggregatorId = Value val Sum: AggregatorId = Value("sum") val Max: AggregatorId = Value("max") val MultiDecay: AggregatorId = Value("multi_decay") } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/README.md ================================================ # Social Graph Candidate Source Provides candidate expansion based on the Twitter social graph. Currently, the expansion is mainly based on the "follow" social graph edge, which allows the service to identify recent following accounts for a given set of accounts. The input accounts could be a user's recent following, engaged, or other related accounts. In summary, the Social Graph Candidate Source utilizes the Twitter social graph to provide candidate expansion, primarily focusing on recent following accounts. ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/RecentFollowingRecentFollowingExpansionSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.socialgraph import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.candidate_sources.base.TwoHopExpansionCandidateSource import com.twitter.follow_recommendations.common.clients.socialgraph.RecentEdgesQuery import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient import com.twitter.follow_recommendations.common.models.AccountProof import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FollowProof import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds import com.twitter.follow_recommendations.common.models.Reason import com.twitter.hermit.model.Algorithm import com.twitter.inject.Logging import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.socialgraph.thriftscala.RelationshipType import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import javax.inject.Inject import javax.inject.Singleton /** * This candidate source is a two hop expansion over the follow graph. The candidates returned from this source is the users that get followed by the target user's recent followings. It will call SocialGraph `n` + 1 times where `n` is the number of recent followings of the target user to be considered. */ @Singleton class RecentFollowingRecentFollowingExpansionSource @Inject() ( socialGraphClient: SocialGraphClient, statsReceiver: StatsReceiver) extends TwoHopExpansionCandidateSource[ HasParams with HasRecentFollowedUserIds, Long, Long, CandidateUser ] with Logging { override val identifier: CandidateSourceIdentifier = RecentFollowingRecentFollowingExpansionSource.Identifier val stats = statsReceiver.scope(identifier.name) override def firstDegreeNodes( target: HasParams with HasRecentFollowedUserIds ): Stitch[Seq[Long]] = Stitch.value( target.recentFollowedUserIds .getOrElse(Nil).take( RecentFollowingRecentFollowingExpansionSource.NumFirstDegreeNodesToRetrieve) ) override def secondaryDegreeNodes( target: HasParams with HasRecentFollowedUserIds, node: Long ): Stitch[Seq[Long]] = socialGraphClient .getRecentEdgesCached( RecentEdgesQuery( node, Seq(RelationshipType.Following), Some(RecentFollowingRecentFollowingExpansionSource.NumSecondDegreeNodesToRetrieve)), useCachedStratoColumn = target.params(RecentFollowingRecentFollowingExpansionSourceParams.CallSgsCachedColumn) ).map( _.take(RecentFollowingRecentFollowingExpansionSource.NumSecondDegreeNodesToRetrieve)).rescue { case exception: Exception => logger.warn( s"${this.getClass} fails to retrieve second degree nodes for first degree node $node", exception) stats.counter("second_degree_expansion_error").incr() Stitch.Nil } override def aggregateAndScore( target: HasParams with HasRecentFollowedUserIds, firstDegreeToSecondDegreeNodesMap: Map[Long, Seq[Long]] ): Stitch[Seq[CandidateUser]] = { val zipped = firstDegreeToSecondDegreeNodesMap.toSeq.flatMap { case (firstDegreeId, secondDegreeIds) => secondDegreeIds.map(secondDegreeId => firstDegreeId -> secondDegreeId) } val candidateAndConnections = zipped .groupBy { case (_, secondDegreeId) => secondDegreeId } .mapValues { v => v.map { case (firstDegreeId, _) => firstDegreeId } } .toSeq .sortBy { case (_, connections) => -connections.size } .map { case (candidateId, connections) => CandidateUser( id = candidateId, score = Some(CandidateUser.DefaultCandidateScore), reason = Some( Reason( Some(AccountProof(followProof = Some(FollowProof(connections, connections.size)))))) ).withCandidateSource(identifier) } Stitch.value(candidateAndConnections) } } object RecentFollowingRecentFollowingExpansionSource { val Identifier = CandidateSourceIdentifier(Algorithm.NewFollowingNewFollowingExpansion.toString) val NumFirstDegreeNodesToRetrieve = 5 val NumSecondDegreeNodesToRetrieve = 20 } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/RecentFollowingRecentFollowingExpansionSourceFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.socialgraph import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @Singleton class RecentFollowingRecentFollowingExpansionSourceFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[FSParam[Boolean] with FSName] = Seq( RecentFollowingRecentFollowingExpansionSourceParams.CallSgsCachedColumn, ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/RecentFollowingRecentFollowingExpansionSourceParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.socialgraph import com.twitter.timelines.configapi.FSParam object RecentFollowingRecentFollowingExpansionSourceParams { object CallSgsCachedColumn extends FSParam[Boolean]( "recent_following_recent_following_source_call_sgs_cached_column", true) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "src/scala/com/twitter/onboarding/relevance/features/strongtie", "src/thrift/com/twitter/search/account_search/extended_network:extended_network_users-scala", "strato/config/columns/hub:hub-strato-client", "strato/config/columns/onboarding/userrecs:userrecs-strato-client", "strato/config/columns/search/account_search:account_search-strato-client", "strato/src/main/scala/com/twitter/strato/client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/BaseOnlineSTPSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds import com.twitter.follow_recommendations.common.models.STPGraph import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.util.logging.Logging import com.twitter.wtf.scalding.jobs.strong_tie_prediction.STPFeatureGenerator import com.twitter.wtf.scalding.jobs.strong_tie_prediction.STPRecord abstract class BaseOnlineSTPSource( stpGraphBuilder: STPGraphBuilder, baseStatsReceiver: StatsReceiver) extends CandidateSource[ HasClientContext with HasParams with HasRecentFollowedUserIds, CandidateUser ] with Logging { protected val statsReceiver: StatsReceiver = baseStatsReceiver.scope("online_stp") override val identifier: CandidateSourceIdentifier = BaseOnlineSTPSource.Identifier def getCandidates( records: Seq[STPRecord], request: HasClientContext with HasParams with HasRecentFollowedUserIds ): Stitch[Seq[CandidateUser]] override def apply( request: HasClientContext with HasParams with HasRecentFollowedUserIds ): Stitch[Seq[CandidateUser]] = request.getOptionalUserId .map { userId => stpGraphBuilder(request) .flatMap { graph: STPGraph => logger.debug(graph) val records = STPFeatureGenerator.constructFeatures( userId, graph.firstDegreeEdgeInfoList, graph.secondDegreeEdgeInfoList) getCandidates(records, request) } }.getOrElse(Stitch.Nil) } object BaseOnlineSTPSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.OnlineStrongTiePredictionRecNoCaching.toString) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/Dbv2StpScorer.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.twitter.cortex.deepbird.runtime.prediction_engine.TensorflowPredictionEngine import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.ml.api.Feature.Continuous import com.twitter.ml.api.util.SRichDataRecord import com.twitter.ml.prediction_service.PredictionRequest import com.twitter.stitch.Stitch import com.twitter.wtf.scalding.jobs.strong_tie_prediction.STPRecord import com.twitter.wtf.scalding.jobs.strong_tie_prediction.STPRecordAdapter import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton /** * STP ML ranker trained using DeepBirdV2 */ @Singleton class Dbv2StpScorer @Inject() ( @Named(GuiceNamedConstants.STP_DBV2_SCORER) tfPredictionEngine: TensorflowPredictionEngine) { def getScoredResponse(record: STPRecord): Stitch[Option[Double]] = { val request: PredictionRequest = new PredictionRequest( STPRecordAdapter.adaptToDataRecord(record)) val responseStitch = Stitch.callFuture(tfPredictionEngine.getPrediction(request)) responseStitch.map { response => val richDr = SRichDataRecord(response.getPrediction) richDr.getFeatureValueOpt(new Continuous("output")) } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/EpStpScorer.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.bijection.thrift.BinaryThriftCodec import com.twitter.relevance.ep_model.scorer.EPScorer import com.twitter.relevance.ep_model.scorer.ScorerUtil import com.twitter.relevance.ep_model.thrift import com.twitter.relevance.ep_model.thriftscala.EPScoringOptions import com.twitter.relevance.ep_model.thriftscala.EPScoringRequest import com.twitter.relevance.ep_model.thriftscala.EPScoringResponse import com.twitter.relevance.ep_model.thriftscala.Record import com.twitter.stitch.Stitch import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ import scala.util.Success case class ScoredResponse(score: Double, featuresBreakdown: Option[String] = None) /** * STP ML ranker trained using prehistoric ML framework */ @Singleton class EpStpScorer @Inject() (epScorer: EPScorer) { private def getScore(responses: List[EPScoringResponse]): Option[ScoredResponse] = responses.headOption .flatMap { response => response.scores.flatMap { _.headOption.map(score => ScoredResponse(ScorerUtil.normalize(score))) } } def getScoredResponse( record: Record, details: Boolean = false ): Stitch[Option[ScoredResponse]] = { val scoringOptions = EPScoringOptions( addFeaturesBreakDown = details, addTransformerIntermediateRecords = details ) val request = EPScoringRequest(auxFeatures = Some(Seq(record)), options = Some(scoringOptions)) Stitch.callFuture( BinaryThriftCodec[thrift.EPScoringRequest] .invert(BinaryScalaCodec(EPScoringRequest).apply(request)) .map { thriftRequest: thrift.EPScoringRequest => val responsesF = epScorer .score(List(thriftRequest).asJava) .map( _.asScala.toList .map(response => BinaryScalaCodec(EPScoringResponse) .invert(BinaryThriftCodec[thrift.EPScoringResponse].apply(response))) .collect { case Success(response) => response } ) responsesF.map(getScore) } .getOrElse(Future(None))) } } object EpStpScorer { val WithFeaturesBreakDown = false } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/MutualFollowStrongTiePredictionSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.twitter.follow_recommendations.common.clients.socialgraph.RecentEdgesQuery import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.socialgraph.thriftscala.RelationshipType import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.onboarding.userrecs.StrongTiePredictionFeaturesOnUserClientColumn import javax.inject.Singleton import javax.inject.Inject /** * Returns mutual follows. It first gets mutual follows from recent 100 follows and followers, and then unions this * with mutual follows from STP features dataset. */ @Singleton class MutualFollowStrongTiePredictionSource @Inject() ( sgsClient: SocialGraphClient, strongTiePredictionFeaturesOnUserClientColumn: StrongTiePredictionFeaturesOnUserClientColumn) extends CandidateSource[HasClientContext with HasRecentFollowedUserIds, CandidateUser] { val identifier: CandidateSourceIdentifier = MutualFollowStrongTiePredictionSource.Identifier override def apply( target: HasClientContext with HasRecentFollowedUserIds ): Stitch[Seq[CandidateUser]] = { target.getOptionalUserId match { case Some(userId) => val newFollowings = target.recentFollowedUserIds .getOrElse(Nil) .take(MutualFollowStrongTiePredictionSource.NumOfRecentFollowings) val newFollowersStitch = sgsClient .getRecentEdges(RecentEdgesQuery(userId, Seq(RelationshipType.FollowedBy))).map( _.take(MutualFollowStrongTiePredictionSource.NumOfRecentFollowers)) val mutualFollowsStitch = strongTiePredictionFeaturesOnUserClientColumn.fetcher .fetch(userId).map(_.v.flatMap(_.topMutualFollows.map(_.map(_.userId))).getOrElse(Nil)) Stitch.join(newFollowersStitch, mutualFollowsStitch).map { case (newFollowers, mutualFollows) => { (newFollowings.intersect(newFollowers) ++ mutualFollows).distinct .map(id => CandidateUser(id, Some(CandidateUser.DefaultCandidateScore))) } } case _ => Stitch.Nil } } } object MutualFollowStrongTiePredictionSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.MutualFollowSTP.toString) val NumOfRecentFollowings = 100 val NumOfRecentFollowers = 100 } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineMutualFollowExpansionSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.google.inject.Singleton import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.strato.generated.client.onboarding.userrecs.MutualFollowExpansionClientColumn import javax.inject.Inject /** * A source that finds the mutual follows of one's mutual follows that one isn't following already. */ @Singleton class OfflineMutualFollowExpansionSource @Inject() ( column: MutualFollowExpansionClientColumn) extends OfflineStrongTiePredictionBaseSource(column.fetcher) { override val identifier: CandidateSourceIdentifier = OfflineMutualFollowExpansionSource.Identifier } object OfflineMutualFollowExpansionSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier(Algorithm.MutualFollowExpansion.toString) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStpSourceFsConfig.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @Singleton class OfflineStpSourceFsConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[FSParam[Boolean] with FSName] = Seq( OfflineStpSourceParams.UseDenserPmiMatrix ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStpSourceParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.twitter.timelines.configapi.FSParam object OfflineStpSourceParams { // If enabled, we use the new, denser version of PMI matrix to generate OfflineSTP candidates. case object UseDenserPmiMatrix extends FSParam[Boolean]("offline_stp_source_use_denser_pmi_matrix", default = false) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStpSourceWithDensePmiMatrix.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.google.inject.Singleton import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.strato.generated.client.hub.PpmiDenseMatrixCandidatesClientColumn import javax.inject.Inject /** * Main source for strong-tie-prediction candidates generated offline. */ @Singleton class OfflineStpSourceWithDensePmiMatrix @Inject() ( stpColumn: PpmiDenseMatrixCandidatesClientColumn) extends OfflineStrongTiePredictionBaseSource(stpColumn.fetcher) { override val identifier: CandidateSourceIdentifier = OfflineStpSourceWithDensePmiMatrix.Identifier } object OfflineStpSourceWithDensePmiMatrix { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier(Algorithm.StrongTiePredictionRec.toString) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStpSourceWithLegacyPmiMatrix.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.google.inject.Singleton import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.strato.generated.client.onboarding.userrecs.StrongTiePredictionClientColumn import javax.inject.Inject /** * Main source for strong-tie-prediction candidates generated offline. */ @Singleton class OfflineStpSourceWithLegacyPmiMatrix @Inject() ( stpColumn: StrongTiePredictionClientColumn) extends OfflineStrongTiePredictionBaseSource(stpColumn.fetcher) { override val identifier: CandidateSourceIdentifier = OfflineStpSourceWithLegacyPmiMatrix.Identifier } object OfflineStpSourceWithLegacyPmiMatrix { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier(Algorithm.StrongTiePredictionRec.toString) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStrongTiePredictionBaseSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.twitter.follow_recommendations.common.models.AccountProof import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FollowProof import com.twitter.follow_recommendations.common.models.Reason import com.twitter.hermit.stp.thriftscala.STPResult import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.timelines.configapi.HasParams /** Base class that all variants of our offline stp dataset can extend. Assumes the same STPResult * value in the key and converts the result into the necessary internal model. */ abstract class OfflineStrongTiePredictionBaseSource( fetcher: Fetcher[Long, Unit, STPResult]) extends CandidateSource[HasParams with HasClientContext, CandidateUser] { def fetch( target: Long, ): Stitch[Seq[CandidateUser]] = { fetcher .fetch(target) .map { result => result.v .map { candidates => OfflineStrongTiePredictionBaseSource.map(target, candidates) } .getOrElse(Nil) .map(_.withCandidateSource(identifier)) } } override def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] = { request.getOptionalUserId.map(fetch).getOrElse(Stitch.Nil) } } object OfflineStrongTiePredictionBaseSource { def map(target: Long, candidates: STPResult): Seq[CandidateUser] = { for { candidate <- candidates.strongTieUsers.sortBy(-_.score) } yield CandidateUser( id = candidate.userId, score = Some(candidate.score), reason = Some( Reason( Some( AccountProof( followProof = candidate.socialProof.map(proof => FollowProof(proof, proof.size)) ) ) ) ) ) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStrongTiePredictionSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.google.inject.Singleton import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStpSourceParams.UseDenserPmiMatrix import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.util.logging.Logging import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import javax.inject.Inject object OfflineStpScore extends Feature[UserCandidate, Option[Double]] /** * Main source for strong-tie-prediction candidates generated offline. */ @Singleton class OfflineStrongTiePredictionSource @Inject() ( offlineStpSourceWithLegacyPmiMatrix: OfflineStpSourceWithLegacyPmiMatrix, offlineStpSourceWithDensePmiMatrix: OfflineStpSourceWithDensePmiMatrix) extends CandidateSource[HasParams with HasClientContext, CandidateUser] with Logging { override val identifier: CandidateSourceIdentifier = OfflineStrongTiePredictionSource.Identifier override def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] = { if (request.params(UseDenserPmiMatrix)) { logger.info("Using dense PMI matrix.") offlineStpSourceWithDensePmiMatrix(request) } else { logger.info("Using legacy PMI matrix.") offlineStpSourceWithLegacyPmiMatrix(request) } } } object OfflineStrongTiePredictionSource { val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier(Algorithm.StrongTiePredictionRec.toString) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @Singleton class OnlineSTPSourceFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[FSParam[Boolean] with FSName] = Seq( OnlineSTPSourceParams.DisableHeavyRanker, OnlineSTPSourceParams.UseDBv2Scorer, ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.Param object OnlineSTPSourceParams { // This replaces the old scorer module, located at EpStpScorer.scala, with the new scorer, located // at Dbv2StpScorer.scala. case object UseDBv2Scorer extends FSParam[Boolean]("online_stp_source_dbv2_scorer_enabled", default = false) // For experiments that test the impact of an improved OnlineSTP source, this controls the usage // of the PostNux heavy-ranker. Note that this FS should *NOT* trigger bucket impressions. case object DisableHeavyRanker extends FSParam[Boolean]("online_stp_source_disable_heavy_ranker", default = false) case object SetPredictionDetails extends Param[Boolean](default = false) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceScorer.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import javax.inject.Inject import javax.inject.Singleton @Singleton class OnlineSTPSourceScorer @Inject() ( onlineSTPSourceWithEPScorer: OnlineSTPSourceWithEPScorer) extends CandidateSource[ HasClientContext with HasParams with HasRecentFollowedUserIds, CandidateUser ] { override def apply( request: HasClientContext with HasParams with HasRecentFollowedUserIds ): Stitch[Seq[CandidateUser]] = { onlineSTPSourceWithEPScorer(request) } override val identifier: CandidateSourceIdentifier = BaseOnlineSTPSource.Identifier } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceWithDeepbirdV2Scorer.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.models.AccountProof import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FollowProof import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds import com.twitter.follow_recommendations.common.models.Reason import com.twitter.onboarding.relevance.features.strongtie.{ StrongTieFeatures => StrongTieFeaturesWrapper } import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.wtf.scalding.jobs.strong_tie_prediction.STPRecord import javax.inject.Inject import javax.inject.Singleton @Singleton class OnlineSTPSourceWithDeepbirdV2Scorer @Inject() ( dbv2StpScorer: Dbv2StpScorer, stpGraphBuilder: STPGraphBuilder, baseStatReceiver: StatsReceiver) extends BaseOnlineSTPSource(stpGraphBuilder, baseStatReceiver) { private val dbv2ScorerUsedCounter = statsReceiver.counter("dbv2_scorer_used") private val dbv2ScorerFailureCounter = statsReceiver.counter("dbv2_scorer_failure") private val dbv2ScorerSuccessCounter = statsReceiver.counter("dbv2_scorer_success") override def getCandidates( records: Seq[STPRecord], request: HasClientContext with HasParams with HasRecentFollowedUserIds, ): Stitch[Seq[CandidateUser]] = { val possibleCandidates: Seq[Stitch[Option[CandidateUser]]] = records.map { trainingRecord => dbv2ScorerUsedCounter.incr() val score = dbv2StpScorer.getScoredResponse(trainingRecord) score.map { case None => dbv2ScorerFailureCounter.incr() None case Some(scoreVal) => dbv2ScorerSuccessCounter.incr() Some( CandidateUser( id = trainingRecord.destinationId, score = Some(OnlineSTPSourceWithDeepbirdV2Scorer.logitSubtraction(scoreVal)), reason = Some( Reason(Some( AccountProof(followProof = Some(FollowProof(trainingRecord.socialProof, trainingRecord.socialProof.size))) ))) ).withCandidateSourceAndFeatures( identifier, Seq(StrongTieFeaturesWrapper(trainingRecord.features))) ) } } Stitch.collect(possibleCandidates).map { _.flatten.sortBy(-_.score.getOrElse(0.0)) } } } object OnlineSTPSourceWithDeepbirdV2Scorer { // The following two variables are the means for the distribution of scores coming from the legacy // and DBv2 OnlineSTP models. We need this to calibrate the DBv2 scores and align the two means. // BQ Link: https://console.cloud.google.com/bigquery?sq=213005704923:e06ac27e4db74385a77a4b538c531f82 private val legacyMeanScore = 0.0478208871192468 private val dbv2MeanScore = 0.238666097210261 // In below are the necessary functions to calibrate the scores such that the means are aligned. private val EPS: Double = 1e-8 private val e: Double = math.exp(1) private def sigmoid(x: Double): Double = math.pow(e, x) / (math.pow(e, x) + 1) // We add an EPS to the denominator to avoid division by 0. private def logit(x: Double): Double = math.log(x / (1 - x + EPS)) def logitSubtraction(x: Double): Double = sigmoid( logit(x) - (logit(dbv2MeanScore) - logit(legacyMeanScore))) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceWithEPScorer.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.candidate_sources.stp.OnlineSTPSourceParams.SetPredictionDetails import com.twitter.follow_recommendations.common.models.AccountProof import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FollowProof import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds import com.twitter.follow_recommendations.common.models.Reason import com.twitter.onboarding.relevance.features.strongtie.{ StrongTieFeatures => StrongTieFeaturesWrapper } import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.util.logging.Logging import com.twitter.wtf.scalding.jobs.strong_tie_prediction.STPRecord import javax.inject.Inject import javax.inject.Singleton @Singleton class OnlineSTPSourceWithEPScorer @Inject() ( epStpScorer: EpStpScorer, stpGraphBuilder: STPGraphBuilder, baseStatReceiver: StatsReceiver) extends BaseOnlineSTPSource(stpGraphBuilder, baseStatReceiver) with Logging { private val epScorerUsedCounter = statsReceiver.counter("ep_scorer_used") override def getCandidates( records: Seq[STPRecord], request: HasClientContext with HasParams with HasRecentFollowedUserIds, ): Stitch[Seq[CandidateUser]] = { epScorerUsedCounter.incr() val possibleCandidates: Seq[Stitch[Option[CandidateUser]]] = records.map { trainingRecord => val scoredResponse = epStpScorer.getScoredResponse(trainingRecord.record, request.params(SetPredictionDetails)) scoredResponse.map(_.map { response: ScoredResponse => logger.debug(response) CandidateUser( id = trainingRecord.destinationId, score = Some(response.score), reason = Some( Reason( Some( AccountProof(followProof = Some(FollowProof(trainingRecord.socialProof, trainingRecord.socialProof.size))) ))) ).withCandidateSourceAndFeatures( identifier, Seq(StrongTieFeaturesWrapper(trainingRecord.features))) }) } Stitch.collect(possibleCandidates).map { _.flatten.sortBy(-_.score.getOrElse(0.0)) } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/README.md ================================================ # Strong Tie Prediction (STP) Candidate Source Provides accounts with a high probability of potential mutual follows between the target user and the candidates. ## STP: Strong Tie Prediction STP refers to the prediction of p(MutualFollow) for a given pair of users, which powers the concept of People You May Know (PYMK). For training, positives are existing mutual follows and negatives are mutual follows of your follows. Features help distinguish between friends and friends-of-friends. For inference, candidates are the topK mutuals of your follows. These are rescored, and we only send the topN to the product or next re-ranker. ### Online STP Online STP generates a pool of candidates who are then ranked via a lightweight ranker. It does this through a two-hop expansion of the mutual follow graph of users, where the first-degree neighbor is another user who has a link with the target user from following types: * Mutual Follow * Outbound phone contacts * Outbound email contacts * Inbound phone contacts * Inbound email contacts The second-degree neighbor can only be a mutual follow link. Currently, online STP can only perform the two-hop expansions on new users (<= 30 days since signup) due to compute resource constraints. Features used for the lightweight ranker: * realGraphWeight: real graph weight between user and first degree nodes * isForwardEmail: whether the candidate is in the user's email book * isReverseEmail: whether the user is in the candidate's email book * isForwardPhonebook: whether the candidate is in the user's phone book * isReversePhonebook: whether the user is in the candidate's phone book * numMutualFollowPath: number of mutual follow path between the user and the candidate * numLowTweepcredFollowPath: number of mutual low TweepCred path between the user and the candidate * Tweepcred is a social network analysis tool that calculates the influence of Twitter users based on their interactions with other users. The tool uses the PageRank algorithm to rank users based on their influence. * hasForwardEmailPath: is there a third user x in the user's email book that connect user -> x -> candidate? * hasReverseEmailPath: is there a third user x in the user's reverse email book that connect user -> x -> candidate? * hasForwardPhonebookPath: is there a third user x in the user's phonebook that connect user -> x -> candidate? * hasReversePhonebookPath: is there a third user x in the user's reverse phonebook that connect user -> x -> candidate? ### Offline STP Offline STP is powered by Pointwise Mutual Information, which measures the association between two users based on Twitter's mutual follow graph. An offline job generates candidates based on the overlap between their Mutual and Addressbook Follows and that of the target user. Candidates are then made available online. Candidates in OfflineSTP are "accounts that have a high overlap of mutually-followed accounts with an account in your follow graph." This can potentially mean that OfflineSTP has a bigger reach than OnlineSTP. For example, in the provided diagram, B and C have a high overlap of mutual follows, so it would be considered a candidate for A that is three hops away. ![img.png](img.png) Overall, STP is a useful candidate source for generating potential follow recommendations based on strong ties between users, but it should be used in conjunction with other candidate sources and re-rankers to provide a well-rounded set of recommendations for the target user. ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/STPFirstDegreeFetcher.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource import com.twitter.follow_recommendations.common.clients.real_time_real_graph.RealTimeRealGraphClient import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds import com.twitter.follow_recommendations.common.models.PotentialFirstDegreeEdge import com.twitter.follow_recommendations.common.stores.LowTweepCredFollowStore import com.twitter.hermit.model.Algorithm import com.twitter.hermit.model.Algorithm.Algorithm import com.twitter.inject.Logging import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.util.Duration import com.twitter.util.Timer import com.twitter.wtf.scalding.jobs.strong_tie_prediction.FirstDegreeEdge import com.twitter.wtf.scalding.jobs.strong_tie_prediction.FirstDegreeEdgeInfo import com.twitter.wtf.scalding.jobs.strong_tie_prediction.FirstDegreeEdgeInfoMonoid import javax.inject.Inject import javax.inject.Singleton // Grabs FirstDegreeNodes from Candidate Sources @Singleton class STPFirstDegreeFetcher @Inject() ( realTimeGraphClient: RealTimeRealGraphClient, reversePhoneBookSource: ReversePhoneBookSource, reverseEmailBookSource: ReverseEmailBookSource, forwardEmailBookSource: ForwardEmailBookSource, forwardPhoneBookSource: ForwardPhoneBookSource, mutualFollowStrongTiePredictionSource: MutualFollowStrongTiePredictionSource, lowTweepCredFollowStore: LowTweepCredFollowStore, timer: Timer, statsReceiver: StatsReceiver) extends Logging { private val stats = statsReceiver.scope("STPFirstDegreeFetcher") private val stitchRequests = stats.scope("stitchRequests") private val allStitchRequests = stitchRequests.counter("all") private val timeoutStitchRequests = stitchRequests.counter("timeout") private val successStitchRequests = stitchRequests.counter("success") private implicit val firstDegreeEdgeInfoMonoid: FirstDegreeEdgeInfoMonoid = new FirstDegreeEdgeInfoMonoid /** * Used to map from algorithm to the correct fetcher and firstDegreeEdgeInfo. * Afterward, uses fetcher to get candidates and construct the correct FirstDegreeEdgeInfo. * */ private def getPotentialFirstEdgesFromFetcher( userId: Long, target: HasClientContext with HasParams with HasRecentFollowedUserIds, algorithm: Algorithm, weight: Double ): Stitch[Seq[PotentialFirstDegreeEdge]] = { val (candidates, edgeInfo) = algorithm match { case Algorithm.MutualFollowSTP => ( mutualFollowStrongTiePredictionSource(target), Some(FirstDegreeEdgeInfo(mutualFollow = true))) case Algorithm.ReverseEmailBookIbis => (reverseEmailBookSource(target), Some(FirstDegreeEdgeInfo(reverseEmail = true))) case Algorithm.ReversePhoneBook => (reversePhoneBookSource(target), Some(FirstDegreeEdgeInfo(reversePhone = true))) case Algorithm.ForwardEmailBook => (forwardEmailBookSource(target), Some(FirstDegreeEdgeInfo(forwardEmail = true))) case Algorithm.ForwardPhoneBook => (forwardPhoneBookSource(target), Some(FirstDegreeEdgeInfo(forwardPhone = true))) case Algorithm.LowTweepcredFollow => ( lowTweepCredFollowStore.getLowTweepCredUsers(target), Some(FirstDegreeEdgeInfo(lowTweepcredFollow = true))) case _ => (Stitch.Nil, None) } candidates.map(_.flatMap { candidate => edgeInfo.map(PotentialFirstDegreeEdge(userId, candidate.id, algorithm, weight, _)) }) } /** * Using the DefaultMap (AlgorithmToScore) we iterate through algorithm/weights to get * candidates with a set weight. Then, given repeating candidates (by candidate id). * Given those candidates we group by the candidateId and sum all below weights and combine * the edgeInfos of into one. Then we choose the candidates with most weight. Finally, * we attach the realGraphWeight score to those candidates. * */ def getFirstDegreeEdges( target: HasClientContext with HasParams with HasRecentFollowedUserIds ): Stitch[Seq[FirstDegreeEdge]] = { target.getOptionalUserId .map { userId => allStitchRequests.incr() val firstEdgesQueryStitch = Stitch .collect(STPFirstDegreeFetcher.DefaultGraphBuilderAlgorithmToScore.map { case (algorithm, candidateWeight) => getPotentialFirstEdgesFromFetcher(userId, target, algorithm, candidateWeight) }.toSeq) .map(_.flatten) val destinationIdsToEdges = firstEdgesQueryStitch .map(_.groupBy(_.connectingId).map { case (destinationId: Long, edges: Seq[PotentialFirstDegreeEdge]) => val combinedDestScore = edges.map(_.score).sum val combinedEdgeInfo: FirstDegreeEdgeInfo = edges.map(_.edgeInfo).fold(firstDegreeEdgeInfoMonoid.zero) { (aggregatedInfo, currentInfo) => firstDegreeEdgeInfoMonoid.plus(aggregatedInfo, currentInfo) } (destinationId, combinedEdgeInfo, combinedDestScore) }).map(_.toSeq) val topDestinationEdges = destinationIdsToEdges.map(_.sortBy { case (_, _, combinedDestScore) => -combinedDestScore }.take(STPFirstDegreeFetcher.MaxNumFirstDegreeEdges)) Stitch .join(realTimeGraphClient.getRealGraphWeights(userId), topDestinationEdges).map { case (realGraphWeights, topDestinationEdges) => successStitchRequests.incr() topDestinationEdges.map { case (destinationId, combinedEdgeInfo, _) => val updatedEdgeInfo = combinedEdgeInfo.copy( realGraphWeight = realGraphWeights.getOrElse(destinationId, 0.0), lowTweepcredFollow = !combinedEdgeInfo.mutualFollow && combinedEdgeInfo.lowTweepcredFollow ) FirstDegreeEdge(userId, destinationId, updatedEdgeInfo) } }.within(STPFirstDegreeFetcher.LongTimeoutFetcher)(timer).rescue { case ex => timeoutStitchRequests.incr() logger.error("Exception while loading direct edges in OnlineSTP: ", ex) Stitch.Nil } }.getOrElse(Stitch.Nil) } } object STPFirstDegreeFetcher { val MaxNumFirstDegreeEdges = 200 val DefaultGraphBuilderAlgorithmToScore = Map( Algorithm.MutualFollowSTP -> 10.0, Algorithm.LowTweepcredFollow -> 6.0, Algorithm.ForwardEmailBook -> 7.0, Algorithm.ForwardPhoneBook -> 9.0, Algorithm.ReverseEmailBookIbis -> 5.0, Algorithm.ReversePhoneBook -> 8.0 ) val LongTimeoutFetcher: Duration = 300.millis } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/STPGraphBuilder.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.twitter.finagle.stats.Stat import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds import com.twitter.follow_recommendations.common.models.STPGraph import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import javax.inject.Inject import javax.inject.Singleton @Singleton class STPGraphBuilder @Inject() ( stpFirstDegreeFetcher: STPFirstDegreeFetcher, stpSecondDegreeFetcher: STPSecondDegreeFetcher, statsReceiver: StatsReceiver) { private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getSimpleName) private val firstDegreeStat: Stat = stats.stat("first_degree_edges") private val secondDegreeStat: Stat = stats.stat("second_degree_edges") def apply( target: HasClientContext with HasParams with HasRecentFollowedUserIds ): Stitch[STPGraph] = stpFirstDegreeFetcher .getFirstDegreeEdges(target).flatMap { firstDegreeEdges => firstDegreeStat.add(firstDegreeEdges.size) stpSecondDegreeFetcher .getSecondDegreeEdges(target, firstDegreeEdges).map { secondDegreeEdges => secondDegreeStat.add(firstDegreeEdges.size) STPGraph(firstDegreeEdges.toList, secondDegreeEdges.toList) } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/STPSecondDegreeFetcher.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.twitter.follow_recommendations.common.models.IntermediateSecondDegreeEdge import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.onboarding.userrecs.StrongTiePredictionFeaturesOnUserClientColumn import com.twitter.timelines.configapi.HasParams import com.twitter.wtf.scalding.jobs.strong_tie_prediction.FirstDegreeEdge import com.twitter.wtf.scalding.jobs.strong_tie_prediction.SecondDegreeEdge import com.twitter.wtf.scalding.jobs.strong_tie_prediction.SecondDegreeEdgeInfo import javax.inject.Inject import javax.inject.Singleton // Link to code functionality we're migrating @Singleton class STPSecondDegreeFetcher @Inject() ( strongTiePredictionFeaturesOnUserClientColumn: StrongTiePredictionFeaturesOnUserClientColumn) { private def scoreSecondDegreeEdge(edge: SecondDegreeEdge): (Int, Int, Int) = { def bool2int(b: Boolean): Int = if (b) 1 else 0 ( -edge.edgeInfo.numMutualFollowPath, -edge.edgeInfo.numLowTweepcredFollowPath, -(bool2int(edge.edgeInfo.forwardEmailPath) + bool2int(edge.edgeInfo.reverseEmailPath) + bool2int(edge.edgeInfo.forwardPhonePath) + bool2int(edge.edgeInfo.reversePhonePath)) ) } // Use each first-degree edge(w/ candidateId) to expand and find mutual follows. // Then, with the mutual follows, group-by candidateId and join edge information // to create secondDegree edges. def getSecondDegreeEdges( target: HasClientContext with HasParams, firstDegreeEdges: Seq[FirstDegreeEdge] ): Stitch[Seq[SecondDegreeEdge]] = { target.getOptionalUserId .map { userId => val firstDegreeConnectingIds = firstDegreeEdges.map(_.dstId) val firstDegreeEdgeInfoMap = firstDegreeEdges.map(e => (e.dstId, e.edgeInfo)).toMap val intermediateSecondDegreeEdgesStitch = Stitch .traverse(firstDegreeConnectingIds) { connectingId => val stpFeaturesOptStitch = strongTiePredictionFeaturesOnUserClientColumn.fetcher .fetch(connectingId) .map(_.v) stpFeaturesOptStitch.map { stpFeatureOpt => val intermediateSecondDegreeEdges = for { edgeInfo <- firstDegreeEdgeInfoMap.get(connectingId) stpFeatures <- stpFeatureOpt topSecondDegreeUserIds = stpFeatures.topMutualFollows .getOrElse(Nil) .map(_.userId) .take(STPSecondDegreeFetcher.MaxNumOfMutualFollows) } yield topSecondDegreeUserIds.map( IntermediateSecondDegreeEdge(connectingId, _, edgeInfo)) intermediateSecondDegreeEdges.getOrElse(Nil) } }.map(_.flatten) intermediateSecondDegreeEdgesStitch.map { intermediateSecondDegreeEdges => val secondaryDegreeEdges = intermediateSecondDegreeEdges.groupBy(_.candidateId).map { case (candidateId, intermediateEdges) => SecondDegreeEdge( srcId = userId, dstId = candidateId, edgeInfo = SecondDegreeEdgeInfo( numMutualFollowPath = intermediateEdges.count(_.edgeInfo.mutualFollow), numLowTweepcredFollowPath = intermediateEdges.count(_.edgeInfo.lowTweepcredFollow), forwardEmailPath = intermediateEdges.exists(_.edgeInfo.forwardEmail), reverseEmailPath = intermediateEdges.exists(_.edgeInfo.reverseEmail), forwardPhonePath = intermediateEdges.exists(_.edgeInfo.forwardPhone), reversePhonePath = intermediateEdges.exists(_.edgeInfo.reversePhone), socialProof = intermediateEdges .filter { e => e.edgeInfo.mutualFollow || e.edgeInfo.lowTweepcredFollow } .sortBy(-_.edgeInfo.realGraphWeight) .take(3) .map { c => (c.connectingId, c.edgeInfo.realGraphWeight) } ) ) } secondaryDegreeEdges.toSeq .sortBy(scoreSecondDegreeEdge) .take(STPSecondDegreeFetcher.MaxNumSecondDegreeEdges) } }.getOrElse(Stitch.Nil) } } object STPSecondDegreeFetcher { val MaxNumSecondDegreeEdges = 200 val MaxNumOfMutualFollows = 50 } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/SocialProofEnforcedOfflineStrongTiePredictionSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.stp import com.google.inject.Singleton import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.candidate_sources.base.SocialProofEnforcedCandidateSource import com.twitter.follow_recommendations.common.transforms.modify_social_proof.ModifySocialProof import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import javax.inject.Inject @Singleton class SocialProofEnforcedOfflineStrongTiePredictionSource @Inject() ( offlineStrongTiePredictionSource: OfflineStrongTiePredictionSource, modifySocialProof: ModifySocialProof, statsReceiver: StatsReceiver) extends SocialProofEnforcedCandidateSource( offlineStrongTiePredictionSource, modifySocialProof, SocialProofEnforcedOfflineStrongTiePredictionSource.MinNumSocialProofsRequired, SocialProofEnforcedOfflineStrongTiePredictionSource.Identifier, statsReceiver) object SocialProofEnforcedOfflineStrongTiePredictionSource { val Identifier = CandidateSourceIdentifier( Algorithm.StrongTiePredictionRecWithSocialProof.toString) val MinNumSocialProofsRequired = 1 } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "hermit/hermit-core/src/main/scala/com/twitter/hermit/model", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", "src/thrift/com/twitter/onboarding/relevance/organic_follows_accounts:organic_follows_accounts-scala", "strato/config/columns/onboarding/userrecs:userrecs-strato-client", "strato/src/main/scala/com/twitter/strato/client", "util/util-core/src/main/scala/com/twitter/conversions", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/README.md ================================================ # Top Organic Follows Accounts Provides the most organically followed (i.e. not followed through the Who-To-Follow module) accounts for a given country. ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/TopOrganicFollowsAccountsFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.Param import javax.inject.Inject import javax.inject.Singleton @Singleton class TopOrganicFollowsAccountsFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( TopOrganicFollowsAccountsParams.CandidateSourceEnabled, ) override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( TopOrganicFollowsAccountsParams.CandidateSourceWeight, ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/TopOrganicFollowsAccountsParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSEnumSeqParam import com.twitter.timelines.configapi.FSParam object TopOrganicFollowsAccountsParams { // whether or not to fetch TopOrganicFollowsAccounts candidate sources case object CandidateSourceEnabled extends FSParam[Boolean]("top_organic_follows_accounts_candidate_source_enabled", false) /** * Contains the logic key for account filtering and ranking. Currently we have 3 main logic keys * - new_organic_follows: filtering top organically followed accounts followed by new users * - non_new_organic_follows: filtering top organically followed accounts followed by non new users * - organic_follows: filtering top organically followed accounts followed by all users * Mapping of the Logic Id to Logic key is done via @enum AccountsFilteringAndRankingLogic */ case object AccountsFilteringAndRankingLogics extends FSEnumSeqParam[AccountsFilteringAndRankingLogicId.type]( name = "top_organic_follows_accounts_filtering_and_ranking_logic_ids", default = Seq(AccountsFilteringAndRankingLogicId.OrganicFollows), enum = AccountsFilteringAndRankingLogicId) case object CandidateSourceWeight extends FSBoundedParam[Double]( "top_organic_follows_accounts_candidate_source_weight", default = 1200, min = 0.001, max = 2000) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/TopOrganicFollowsAccountsSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts import com.twitter.escherbird.util.stitchcache.StitchCache import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsParams.AccountsFilteringAndRankingLogics import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsParams.CandidateSourceEnabled import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasGeohashAndCountryCode import com.twitter.hermit.model.Algorithm import com.twitter.onboarding.relevance.organic_follows_accounts.thriftscala.OrganicFollowsAccounts import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.onboarding.userrecs.OrganicFollowsAccountsClientColumn import com.twitter.timelines.configapi.HasParams import com.twitter.util.Duration import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton object AccountsFilteringAndRankingLogicId extends Enumeration { type AccountsFilteringAndRankingLogicId = Value val NewOrganicFollows: AccountsFilteringAndRankingLogicId = Value("new_organic_follows") val NonNewOrganicFollows: AccountsFilteringAndRankingLogicId = Value("non_new_organic_follows") val OrganicFollows: AccountsFilteringAndRankingLogicId = Value("organic_follows") } object TopOrganicFollowsAccountsSource { val MaxCacheSize = 500 val CacheTTL: Duration = Duration.fromHours(24) type Target = HasParams with HasClientContext with HasGeohashAndCountryCode val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.OrganicFollowAccounts.toString) } @Singleton class TopOrganicFollowsAccountsSource @Inject() ( organicFollowsAccountsClientColumn: OrganicFollowsAccountsClientColumn, statsReceiver: StatsReceiver, ) extends CandidateSource[TopOrganicFollowsAccountsSource.Target, CandidateUser] with Logging { /** @see [[CandidateSourceIdentifier]] */ override val identifier: CandidateSourceIdentifier = TopOrganicFollowsAccountsSource.Identifier private val stats = statsReceiver.scope(identifier.name) private val requestsStats = stats.counter("requests") private val noCountryCodeStats = stats.counter("no_country_code") private val successStats = stats.counter("success") private val errorStats = stats.counter("error") private val cache = StitchCache[String, Option[OrganicFollowsAccounts]]( maxCacheSize = TopOrganicFollowsAccountsSource.MaxCacheSize, ttl = TopOrganicFollowsAccountsSource.CacheTTL, statsReceiver = statsReceiver.scope(identifier.name, "cache"), underlyingCall = (k: String) => { organicFollowsAccountsClientColumn.fetcher .fetch(k) .map { result => result.v } } ) /** returns a Seq of ''potential'' content */ override def apply( target: TopOrganicFollowsAccountsSource.Target ): Stitch[Seq[CandidateUser]] = { if (!target.params(CandidateSourceEnabled)) { return Stitch.value(Seq[CandidateUser]()) } requestsStats.incr() target.getCountryCode .orElse(target.geohashAndCountryCode.flatMap(_.countryCode)).map { countryCode => Stitch .collect(target .params(AccountsFilteringAndRankingLogics).map(logic => cache.readThrough(countryCode.toUpperCase() + "-" + logic))) .onSuccess(_ => { successStats.incr() }) .onFailure(t => { debug("candidate source failed identifier = %s".format(identifier), t) errorStats.incr() }) .map(transformOrganicFollowAccountssToCandidateSource) }.getOrElse { noCountryCodeStats.incr() Stitch.value(Seq[CandidateUser]()) } } private def transformOrganicFollowAccountssToCandidateSource( organicFollowsAccounts: Seq[Option[OrganicFollowsAccounts]] ): Seq[CandidateUser] = { organicFollowsAccounts .flatMap(opt => opt .map(accounts => accounts.accounts.map(account => CandidateUser( id = account.accountId, score = Some(account.followedCountScore), ).withCandidateSource(identifier))) .getOrElse(Seq[CandidateUser]())) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "strato/config/columns/onboarding/userrecs:userrecs-strato-client", "strato/src/main/scala/com/twitter/strato/client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/README.md ================================================ # Triangular Loops Candidate Source Provides account candidates based on the graph structures of the form u -> v -> w -> u, where the arrow indicates a follow edge. In other words, it looks for triangular loops in the user-user graph. If the edge v -> u does not exist in the triangular loop, the Triangular Loops Candidate Source recommends u as a potential outbound mutual follow candidate for v. ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/TriangularLoopsFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.triangular_loops import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @Singleton class TriangularLoopsFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[FSParam[Boolean] with FSName] = Nil } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/TriangularLoopsParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.triangular_loops import com.twitter.timelines.configapi.FSParam object TriangularLoopsParams { object KeepOnlyCandidatesWhoFollowTargetUser extends FSParam[Boolean]( "triangular_loops_keep_only_candidates_who_follow_target_user", false) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/TriangularLoopsSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.triangular_loops import com.twitter.follow_recommendations.common.models.AccountProof import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FollowProof import com.twitter.follow_recommendations.common.models.HasRecentFollowedByUserIds import com.twitter.follow_recommendations.common.models.Reason import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.onboarding.userrecs.TriangularLoopsV2OnUserClientColumn import com.twitter.timelines.configapi.HasParams import com.twitter.wtf.triangular_loop.thriftscala.Candidates import javax.inject.Inject import javax.inject.Singleton @Singleton class TriangularLoopsSource @Inject() ( triangularLoopsV2Column: TriangularLoopsV2OnUserClientColumn) extends CandidateSource[ HasParams with HasClientContext with HasRecentFollowedByUserIds, CandidateUser ] { override val identifier: CandidateSourceIdentifier = TriangularLoopsSource.Identifier override def apply( target: HasParams with HasClientContext with HasRecentFollowedByUserIds ): Stitch[Seq[CandidateUser]] = { val candidates = target.getOptionalUserId .map { userId => val fetcher = triangularLoopsV2Column.fetcher fetcher .fetch(userId) .map { result => result.v .map(TriangularLoopsSource.mapCandidatesToCandidateUsers) .getOrElse(Nil) } }.getOrElse(Stitch.Nil) // Make sure recentFollowedByUserIds is populated within the RequestBuilder before enable it if (target.params(TriangularLoopsParams.KeepOnlyCandidatesWhoFollowTargetUser)) filterOutCandidatesNotFollowingTargetUser(candidates, target.recentFollowedByUserIds) else candidates } def filterOutCandidatesNotFollowingTargetUser( candidatesStitch: Stitch[Seq[CandidateUser]], recentFollowings: Option[Seq[Long]] ): Stitch[Seq[CandidateUser]] = { candidatesStitch.map { candidates => val recentFollowingIdsSet = recentFollowings.getOrElse(Nil).toSet candidates.filter(candidate => recentFollowingIdsSet.contains(candidate.id)) } } } object TriangularLoopsSource { val Identifier = CandidateSourceIdentifier(Algorithm.TriangularLoop.toString) val NumResults = 100 def mapCandidatesToCandidateUsers(candidates: Candidates): Seq[CandidateUser] = { candidates.candidates .map { candidate => CandidateUser( id = candidate.incomingUserId, score = Some(1.0 / math .max(1, candidate.numFollowers.getOrElse(0) + candidate.numFollowings.getOrElse(0))), reason = Some( Reason( Some( AccountProof( followProof = if (candidate.socialProofUserIds.isEmpty) None else Some( FollowProof( candidate.socialProofUserIds, candidate.numSocialProof.getOrElse(candidate.socialProofUserIds.size))) ) ) ) ) ).withCandidateSource(Identifier) }.sortBy(-_.score.getOrElse(0.0)).take(NumResults) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", "strato/src/main/scala/com/twitter/strato/client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk/README.md ================================================ # Two-hop Random Walk The TwoHopRandomWalk algorithm re-ranks a user's second degree connections based on recent engagement strength. The algorithm works as follows: * Given a user `src`, find their top K first degree connections `fd(1)`, `fd(2)`, `fd(3)`,...,`fd(K)`. The ranking is based on real graph weights, which measure the recent engagement strength on the edges. * For each of the first degree connections `fd(i)`, expand to their top L connections via real graph, `sd(i,1)`, `sd(i,2)`,...,`sd(i,L)`. Note that sd nodes can also be `src`'s first degree nodes. * Aggregate all the nodes in step 2, filter out the first degree nodes, and calculate the weighted sum for the second degree. * Re-rank the second degree nodes and select the top M results as the algorithm output. ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk/TwoHopRandomWalkSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.two_hop_random_walk import com.twitter.follow_recommendations.common.candidate_sources.base.StratoFetcherWithUnitViewSource import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.strato.client.Fetcher import com.twitter.wtf.candidate.thriftscala.{CandidateSeq => TCandidateSeq} import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class TwoHopRandomWalkSource @Inject() ( @Named(GuiceNamedConstants.TWO_HOP_RANDOM_WALK_FETCHER) fetcher: Fetcher[ Long, Unit, TCandidateSeq ]) extends StratoFetcherWithUnitViewSource[Long, TCandidateSeq]( fetcher, TwoHopRandomWalkSource.Identifier) { override def map(targetUserId: Long, tCandidateSeq: TCandidateSeq): Seq[CandidateUser] = TwoHopRandomWalkSource.map(targetUserId, tCandidateSeq) } object TwoHopRandomWalkSource { def map(targetUserId: Long, tCandidateSeq: TCandidateSeq): Seq[CandidateUser] = { tCandidateSeq.candidates .sortBy(-_.score) .map { tCandidate => CandidateUser(id = tCandidate.userId, score = Some(tCandidate.score)) } } val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier(Algorithm.TwoHopRandomWalk.toString) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "src/thrift/com/twitter/recos/user_user_graph:user_user_graph-scala", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/README.md ================================================ # User-User Graph Candidate Source Provides account candidates generated from the User-User Graph (UUG). ## User-User Graph (UUG) The UUG algorithm reads User-Follow-User engagements that occurred in the past 24-48 hours, and provides accounts that the given user's recent followings have recently followed themselves. The UUG algorithm is implemented using the real-time graph processing library GraphJet. ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/UserUserGraphCandidateSource.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.user_user_graph import com.twitter.finagle.stats.Counter import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.follow_recommendations.common.models._ import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.recos.recos_common.thriftscala.UserSocialProofType import com.twitter.recos.user_user_graph.thriftscala.RecommendUserDisplayLocation import com.twitter.recos.user_user_graph.thriftscala.RecommendUserRequest import com.twitter.recos.user_user_graph.thriftscala.RecommendUserResponse import com.twitter.recos.user_user_graph.thriftscala.RecommendedUser import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.timelines.configapi.HasParams import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class UserUserGraphCandidateSource @Inject() ( @Named(GuiceNamedConstants.USER_USER_GRAPH_FETCHER) fetcher: Fetcher[RecommendUserRequest, Unit, RecommendUserResponse], statsReceiver: StatsReceiver) extends CandidateSource[ UserUserGraphCandidateSource.Target, CandidateUser ] { override val identifier: CandidateSourceIdentifier = UserUserGraphCandidateSource.Identifier val stats: StatsReceiver = statsReceiver.scope("UserUserGraph") val requestCounter: Counter = stats.counter("requests") override def apply( target: UserUserGraphCandidateSource.Target ): Stitch[Seq[CandidateUser]] = { if (target.params(UserUserGraphParams.UserUserGraphCandidateSourceEnabledInWeightMap)) { requestCounter.incr() buildRecommendUserRequest(target) .map { request => fetcher .fetch(request) .map(_.v) .map { responseOpt => responseOpt .map { response => response.recommendedUsers .sortBy(-_.score) .map(convertToCandidateUsers) .map(_.withCandidateSource(identifier)) }.getOrElse(Nil) } }.getOrElse(Stitch.Nil) } else { Stitch.Nil } } private[this] def buildRecommendUserRequest( target: UserUserGraphCandidateSource.Target ): Option[RecommendUserRequest] = { (target.getOptionalUserId, target.recentFollowedUserIds) match { case (Some(userId), Some(recentFollowedUserIds)) => // use recentFollowedUserIds as seeds for initial experiment val seedsWithWeights: Map[Long, Double] = recentFollowedUserIds.map { recentFollowedUserId => recentFollowedUserId -> UserUserGraphCandidateSource.DefaultSeedWeight }.toMap val request = RecommendUserRequest( requesterId = userId, displayLocation = UserUserGraphCandidateSource.DisplayLocation, seedsWithWeights = seedsWithWeights, excludedUserIds = Some(target.excludedUserIds), maxNumResults = Some(target.params.getInt(UserUserGraphParams.MaxCandidatesToReturn)), maxNumSocialProofs = Some(UserUserGraphCandidateSource.MaxNumSocialProofs), minUserPerSocialProof = Some(UserUserGraphCandidateSource.MinUserPerSocialProof), socialProofTypes = Some(Seq(UserUserGraphCandidateSource.SocialProofType)) ) Some(request) case _ => None } } private[this] def convertToCandidateUsers( recommendedUser: RecommendedUser ): CandidateUser = { val socialProofUserIds = recommendedUser.socialProofs.getOrElse(UserUserGraphCandidateSource.SocialProofType, Nil) val reasonOpt = if (socialProofUserIds.nonEmpty) { Some( Reason( Some(AccountProof(followProof = Some(FollowProof(socialProofUserIds, socialProofUserIds.size))))) ) } else { None } CandidateUser( id = recommendedUser.userId, score = Some(recommendedUser.score), reason = reasonOpt) } } object UserUserGraphCandidateSource { type Target = HasParams with HasClientContext with HasRecentFollowedUserIds with HasExcludedUserIds val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( Algorithm.UserUserGraph.toString) //Use HomeTimeline for experiment val DisplayLocation: RecommendUserDisplayLocation = RecommendUserDisplayLocation.HomeTimeLine //Default params used in MagicRecs val DefaultSeedWeight: Double = 1.0 val SocialProofType = UserSocialProofType.Follow val MaxNumSocialProofs = 10 val MinUserPerSocialProof: Map[UserSocialProofType, Int] = Map[UserSocialProofType, Int]((SocialProofType, 2)) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/UserUserGraphFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.user_user_graph import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.Param import javax.inject.Inject import javax.inject.Singleton @Singleton class UserUserGraphFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( UserUserGraphParams.UserUserGraphCandidateSourceEnabledInWeightMap, UserUserGraphParams.UserUserGraphCandidateSourceEnabledInTransform ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/UserUserGraphParams.scala ================================================ package com.twitter.follow_recommendations.common.candidate_sources.user_user_graph import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.Param object UserUserGraphParams { // max number of candidates to return in total, 50 is the default param used in MagicRecs object MaxCandidatesToReturn extends Param[Int](default = 50) // whether or not to include UserUserGraph candidate source in the weighted blending step case object UserUserGraphCandidateSourceEnabledInWeightMap extends FSParam[Boolean]("user_user_graph_candidate_source_enabled_in_weight_map", true) // whether or not to include UserUserGraph candidate source in the final transform step case object UserUserGraphCandidateSourceEnabledInTransform extends FSParam[Boolean]("user_user_graph_candidate_source_enabled_in_transform", true) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/AddressbookClient.scala ================================================ package com.twitter.follow_recommendations.common.clients.addressbook import com.twitter.addressbook.datatypes.thriftscala.QueryType import com.twitter.addressbook.thriftscala.AddressBookGetRequest import com.twitter.addressbook.thriftscala.AddressBookGetResponse import com.twitter.addressbook.thriftscala.Addressbook2 import com.twitter.addressbook.thriftscala.ClientInfo import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.stats.StatsReceiver import com.twitter.wtf.scalding.jobs.addressbook.thriftscala.STPResultFeature import com.twitter.follow_recommendations.common.clients.addressbook.models.Contact import com.twitter.follow_recommendations.common.clients.addressbook.models.EdgeType import com.twitter.follow_recommendations.common.clients.addressbook.models.QueryOption import com.twitter.follow_recommendations.common.clients.addressbook.models.RecordIdentifier import com.twitter.wtf.scalding.jobs.address_book.ABUtil.hashContact import com.twitter.wtf.scalding.jobs.address_book.ABUtil.normalizeEmail import com.twitter.wtf.scalding.jobs.address_book.ABUtil.normalizePhoneNumber import com.twitter.hermit.usercontacts.thriftscala.{UserContacts => tUserContacts} import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import javax.inject.Inject import javax.inject.Singleton @Singleton class AddressbookClient @Inject() ( addressbookService: Addressbook2.MethodPerEndpoint, statsReceiver: StatsReceiver = NullStatsReceiver) { private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getSimpleName) private[this] def getResponseFromService( identifiers: Seq[RecordIdentifier], batchSize: Int, edgeType: EdgeType, maxFetches: Int, queryOption: Option[QueryOption] ): Stitch[Seq[AddressBookGetResponse]] = { Stitch .collect( identifiers.map { identifier => Stitch.callFuture( addressbookService.get(AddressBookGetRequest( clientInfo = ClientInfo(None), identifier = identifier.toThrift, edgeType = edgeType.toThrift, queryType = QueryType.UserId, queryOption = queryOption.map(_.toThrift), maxFetches = maxFetches, resultBatchSize = batchSize ))) } ) } private[this] def getContactsResponseFromService( identifiers: Seq[RecordIdentifier], batchSize: Int, edgeType: EdgeType, maxFetches: Int, queryOption: Option[QueryOption] ): Stitch[Seq[AddressBookGetResponse]] = { Stitch .collect( identifiers.map { identifier => Stitch.callFuture( addressbookService.get(AddressBookGetRequest( clientInfo = ClientInfo(None), identifier = identifier.toThrift, edgeType = edgeType.toThrift, queryType = QueryType.Contact, queryOption = queryOption.map(_.toThrift), maxFetches = maxFetches, resultBatchSize = batchSize ))) } ) } /** Mode of addressbook resolving logic * ManhattanThenABV2: fetching manhattan cached result and backfill with addressbook v2 * ABV2Only: calling addressbook v2 directly without fetching manhattan cached result * This can be controlled by passing a fetcher or not. Passing a fetcher will attempt to use it, * if not then it won't. */ def getUsers( userId: Long, identifiers: Seq[RecordIdentifier], batchSize: Int, edgeType: EdgeType, fetcherOption: Option[Fetcher[Long, Unit, tUserContacts]] = None, maxFetches: Int = 1, queryOption: Option[QueryOption] = None, ): Stitch[Seq[Long]] = { fetcherOption match { case Some(fetcher) => getUsersFromManhattan(userId, fetcher).flatMap { userContacts => if (userContacts.isEmpty) { stats.counter("mhEmptyThenFromAbService").incr() getResponseFromService(identifiers, batchSize, edgeType, maxFetches, queryOption) .map(_.flatMap(_.users).flatten.distinct) } else { stats.counter("fromManhattan").incr() Stitch.value(userContacts) } } case None => stats.counter("fromAbService").incr() getResponseFromService(identifiers, batchSize, edgeType, maxFetches, queryOption) .map(_.flatMap(_.users).flatten.distinct) } } def getHashedContacts( normalizeFn: String => String, extractField: String, )( userId: Long, identifiers: Seq[RecordIdentifier], batchSize: Int, edgeType: EdgeType, fetcherOption: Option[Fetcher[String, Unit, STPResultFeature]] = None, maxFetches: Int = 1, queryOption: Option[QueryOption] = None, ): Stitch[Seq[String]] = { fetcherOption match { case Some(fetcher) => getContactsFromManhattan(userId, fetcher).flatMap { userContacts => if (userContacts.isEmpty) { getContactsResponseFromService( identifiers, batchSize, edgeType, maxFetches, queryOption) .map { response => for { resp <- response contacts <- resp.contacts contactsThrift = contacts.map(Contact.fromThrift) contactsSet = extractField match { case "emails" => contactsThrift.flatMap(_.emails.toSeq.flatten) case "phoneNumbers" => contactsThrift.flatMap(_.phoneNumbers.toSeq.flatten) } hashedAndNormalizedContacts = contactsSet.map(c => hashContact(normalizeFn(c))) } yield hashedAndNormalizedContacts }.map(_.flatten) } else { Stitch.Nil } } case None => { getContactsResponseFromService(identifiers, batchSize, edgeType, maxFetches, queryOption) .map { response => for { resp <- response contacts <- resp.contacts contactsThrift = contacts.map(Contact.fromThrift) contactsSet = extractField match { case "emails" => contactsThrift.flatMap(_.emails.toSeq.flatten) case "phoneNumbers" => contactsThrift.flatMap(_.phoneNumbers.toSeq.flatten) } hashedAndNormalizedContacts = contactsSet.map(c => hashContact(normalizeFn(c))) } yield hashedAndNormalizedContacts }.map(_.flatten) } } } def getEmailContacts = getHashedContacts(normalizeEmail, "emails") _ def getPhoneContacts = getHashedContacts(normalizePhoneNumber, "phoneNumbers") _ private def getUsersFromManhattan( userId: Long, fetcher: Fetcher[Long, Unit, tUserContacts], ): Stitch[Seq[Long]] = fetcher .fetch(userId) .map(_.v.map(_.destinationIds).toSeq.flatten.distinct) private def getContactsFromManhattan( userId: Long, fetcher: Fetcher[String, Unit, STPResultFeature], ): Stitch[Seq[String]] = fetcher .fetch(userId.toString) .map(_.v.map(_.strongTieUserFeature.map(_.destId)).toSeq.flatten.distinct) } object AddressbookClient { val AddressBook2BatchSize = 500 def createQueryOption(edgeType: EdgeType, isPhone: Boolean): Option[QueryOption] = (edgeType, isPhone) match { case (EdgeType.Reverse, _) => None case (EdgeType.Forward, true) => Some( QueryOption( onlyDiscoverableInExpansion = false, onlyConfirmedInExpansion = false, onlyDiscoverableInResult = false, onlyConfirmedInResult = false, fetchGlobalApiNamespace = false, isDebugRequest = false, resolveEmails = false, resolvePhoneNumbers = true )) case (EdgeType.Forward, false) => Some( QueryOption( onlyDiscoverableInExpansion = false, onlyConfirmedInExpansion = false, onlyDiscoverableInResult = false, onlyConfirmedInResult = false, fetchGlobalApiNamespace = false, isDebugRequest = false, resolveEmails = true, resolvePhoneNumbers = false )) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/AddressbookModule.scala ================================================ package com.twitter.follow_recommendations.common.clients.addressbook import com.twitter.addressbook.thriftscala.Addressbook2 import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.follow_recommendations.common.clients.common.BaseClientModule object AddressbookModule extends BaseClientModule[Addressbook2.MethodPerEndpoint] with MtlsClient { override val label = "addressbook" override val dest = "/s/addressbook/addressbook2" } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "addressbook/thrift/src/thrift/com/twitter/addressbook:thrift-scala", "addressbook/thrift/src/thrift/com/twitter/addressbook/datatypes:thrift-scala", "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-thrift-client", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", "src/scala/com/twitter/wtf/scalding/jobs/address_book:ab_util", "src/thrift/com/twitter/hermit/usercontacts:hermit-usercontacts-scala", "src/thrift/com/twitter/wtf/addressbook:addressbook-scala", "stitch/stitch-core", "strato/src/main/scala/com/twitter/strato/client", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "addressbook/thrift/src/thrift/com/twitter/addressbook:thrift-scala", "addressbook/thrift/src/thrift/com/twitter/addressbook/datatypes:thrift-scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/Contact.scala ================================================ package com.twitter.follow_recommendations.common.clients.addressbook.models import com.twitter.addressbook.{thriftscala => t} import com.twitter.util.Time case class Contact( id: Long, emails: Option[Set[String]], phoneNumbers: Option[Set[String]], firstName: Option[String], lastName: Option[String], name: Option[String], appId: Option[Long], appIds: Option[Set[Long]], importedTimestamp: Option[Time]) object Contact { def fromThrift(thriftContact: t.Contact): Contact = Contact( thriftContact.id, thriftContact.emails.map(_.toSet), thriftContact.phoneNumbers.map(_.toSet), thriftContact.firstName, thriftContact.lastName, thriftContact.name, thriftContact.appId, thriftContact.appIds.map(_.toSet), thriftContact.importedTimestamp.map(Time.fromMilliseconds) ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/EdgeType.scala ================================================ package com.twitter.follow_recommendations.common.clients.addressbook.models import com.twitter.addressbook.datatypes.{thriftscala => t} sealed trait EdgeType { def toThrift: t.EdgeType } object EdgeType { case object Forward extends EdgeType { override val toThrift: t.EdgeType = t.EdgeType.Forward } case object Reverse extends EdgeType { override val toThrift: t.EdgeType = t.EdgeType.Reverse } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/QueryOption.scala ================================================ package com.twitter.follow_recommendations.common.clients.addressbook.models import com.twitter.addressbook.{thriftscala => t} case class QueryOption( onlyDiscoverableInExpansion: Boolean, onlyConfirmedInExpansion: Boolean, onlyDiscoverableInResult: Boolean, onlyConfirmedInResult: Boolean, fetchGlobalApiNamespace: Boolean, isDebugRequest: Boolean, resolveEmails: Boolean, resolvePhoneNumbers: Boolean) { def toThrift: t.QueryOption = t.QueryOption( onlyDiscoverableInExpansion, onlyConfirmedInExpansion, onlyDiscoverableInResult, onlyConfirmedInResult, fetchGlobalApiNamespace, isDebugRequest, resolveEmails, resolvePhoneNumbers ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/RecordIdentifier.scala ================================================ package com.twitter.follow_recommendations.common.clients.addressbook.models import com.twitter.addressbook.datatypes.{thriftscala => t} case class RecordIdentifier( userId: Option[Long], email: Option[String], phoneNumber: Option[String]) { def toThrift: t.RecordIdentifier = t.RecordIdentifier(userId, email, phoneNumber) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver/AdRequest.scala ================================================ package com.twitter.follow_recommendations.common.clients.adserver import com.twitter.adserver.{thriftscala => t} import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.product_mixer.core.model.marshalling.request.ClientContext case class AdRequest( clientContext: ClientContext, displayLocation: DisplayLocation, isTest: Option[Boolean], profileUserId: Option[Long]) { def toThrift: t.AdRequestParams = { val request = t.AdRequest( displayLocation = displayLocation.toAdDisplayLocation.getOrElse( throw new MissingAdDisplayLocation(displayLocation)), isTest = isTest, countImpressionsOnCallback = Some(true), numOrganicItems = Some(AdRequest.DefaultNumOrganicItems.toShort), profileUserId = profileUserId ) val clientInfo = t.ClientInfo( clientId = clientContext.appId.map(_.toInt), userIp = clientContext.ipAddress, userId64 = clientContext.userId, guestId = clientContext.guestId, userAgent = clientContext.userAgent, referrer = None, deviceId = clientContext.deviceId, languageCode = clientContext.languageCode, countryCode = clientContext.countryCode ) t.AdRequestParams(request, clientInfo) } } object AdRequest { val DefaultNumOrganicItems = 10 } class MissingAdDisplayLocation(displayLocation: DisplayLocation) extends Exception( s"Display Location ${displayLocation.toString} has no mapped AdsDisplayLocation set.") ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver/AdserverClient.scala ================================================ package com.twitter.follow_recommendations.common.clients.adserver import com.twitter.adserver.thriftscala.NewAdServer import com.twitter.adserver.{thriftscala => t} import com.twitter.stitch.Stitch import javax.inject.{Inject, Singleton} @Singleton class AdserverClient @Inject() (adserverService: NewAdServer.MethodPerEndpoint) { def getAdImpressions(adRequest: AdRequest): Stitch[Seq[t.AdImpression]] = { Stitch .callFuture( adserverService.makeAdRequest(adRequest.toThrift) ).map(_.impressions) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver/AdserverModule.scala ================================================ package com.twitter.follow_recommendations.common.clients.adserver import com.twitter.adserver.thriftscala.NewAdServer import com.twitter.conversions.DurationOps._ import com.twitter.finagle.ThriftMux import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.follow_recommendations.common.clients.common.BaseClientModule object AdserverModule extends BaseClientModule[NewAdServer.MethodPerEndpoint] with MtlsClient { override val label = "adserver" override val dest = "/s/ads/adserver" override def configureThriftMuxClient(client: ThriftMux.Client): ThriftMux.Client = client.withRequestTimeout(500.millis) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-thrift-client", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "src/thrift/com/twitter/ads/adserver:adserver_rpc-scala", "stitch/stitch-core", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "cache/client", "finagle/finagle-memcached/src/main/scala", "finatra/inject/inject-core/src/main/scala", "finatra/inject/inject-thrift-client/src/main/scala", "stitch/stitch-core", "util/util-core:scala", "util/util-thrift", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache/MemcacheClient.scala ================================================ package com.twitter.follow_recommendations.common.clients.cache import com.twitter.bijection.Bijection import com.twitter.conversions.DurationOps._ import com.twitter.finagle.Memcached.Client import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.util.DefaultTimer import com.twitter.io.Buf import com.twitter.stitch.Stitch import com.twitter.util.Duration import com.twitter.util.Future import com.twitter.util.Time import java.security.MessageDigest object MemcacheClient { def apply[V]( client: Client, dest: String, valueBijection: Bijection[Buf, V], ttl: Duration, statsReceiver: StatsReceiver ): MemcacheClient[V] = { new MemcacheClient(client, dest, valueBijection, ttl, statsReceiver) } } class MemcacheClient[V]( client: Client, dest: String, valueBijection: Bijection[Buf, V], ttl: Duration, statsReceiver: StatsReceiver) { val cache = client.newRichClient(dest).adapt[V](valueBijection) val cacheTtl = Time.fromSeconds(ttl.inSeconds) /** * If cache contains key, return value from cache. Otherwise, run the underlying call * to fetch the value, store it in cache, and then return the value. */ def readThrough( key: String, underlyingCall: () => Stitch[V] ): Stitch[V] = { val cachedResult: Stitch[Option[V]] = Stitch .callFuture(getIfPresent(key)) .within(70.millisecond)(DefaultTimer) .rescue { case e: Exception => statsReceiver.scope("rescued").counter(e.getClass.getSimpleName).incr() Stitch(None) } val resultStitch = cachedResult.map { resultOption => resultOption match { case Some(cacheValue) => Stitch.value(cacheValue) case None => val underlyingCallStitch = profileStitch( underlyingCall(), statsReceiver.scope("underlyingCall") ) underlyingCallStitch.map { result => put(key, result) result } } }.flatten // profile the overall Stitch, and return the result profileStitch(resultStitch, statsReceiver.scope("readThrough")) } def getIfPresent(key: String): Future[Option[V]] = { cache .get(hashString(key)) .onSuccess { case Some(value) => statsReceiver.counter("cache_hits").incr() case None => statsReceiver.counter("cache_misses").incr() } .onFailure { case e: Exception => statsReceiver.counter("cache_misses").incr() statsReceiver.scope("rescued").counter(e.getClass.getSimpleName).incr() } .rescue { case _ => Future.None } } def put(key: String, value: V): Future[Unit] = { cache.set(hashString(key), 0, cacheTtl, value) } /** * Hash the input key string to a fixed length format using SHA-256 hash function. */ def hashString(input: String): String = { val bytes = MessageDigest.getInstance("SHA-256").digest(input.getBytes("UTF-8")) bytes.map("%02x".format(_)).mkString } /** * Helper function for timing a stitch, returning the original stitch. * * Defining the profiling function here to keep the dependencies of this class * generic and easy to export (i.e. copy-and-paste) into other services or packages. */ def profileStitch[T](stitch: Stitch[T], stat: StatsReceiver): Stitch[T] = { Stitch .time(stitch) .map { case (response, stitchRunDuration) => stat.counter("requests").incr() stat.stat("latency_ms").add(stitchRunDuration.inMilliseconds) response .onSuccess { _ => stat.counter("success").incr() } .onFailure { e => stat.counter("failures").incr() stat.scope("failures").counter(e.getClass.getSimpleName).incr() } } .lowerFromTry } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache/MemcacheModule.scala ================================================ package com.twitter.follow_recommendations.common.clients.cache import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.Memcached import com.twitter.finagle.Memcached.Client import com.twitter.finagle.mtls.client.MtlsStackClient._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.service.Retries import com.twitter.finagle.service.RetryPolicy import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import javax.inject.Singleton object MemcacheModule extends TwitterModule { @Provides @Singleton def provideMemcacheClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver, ): Client = { Memcached.client .withMutualTls(serviceIdentifier) .withStatsReceiver(statsReceiver.scope("twemcache")) .withTransport.connectTimeout(1.seconds) .withRequestTimeout(1.seconds) .withSession.acquisitionTimeout(10.seconds) .configured(Retries.Policy(RetryPolicy.tries(1))) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache/ThriftBijection.scala ================================================ package com.twitter.follow_recommendations.common.clients.cache import com.twitter.bijection.Bijection import com.twitter.io.Buf import com.twitter.scrooge.CompactThriftSerializer import com.twitter.scrooge.ThriftEnum import com.twitter.scrooge.ThriftStruct import java.nio.ByteBuffer abstract class ThriftBijection[T <: ThriftStruct] extends Bijection[Buf, T] { val serializer: CompactThriftSerializer[T] override def apply(b: Buf): T = { val byteArray = Buf.ByteArray.Owned.extract(b) serializer.fromBytes(byteArray) } override def invert(a: T): Buf = { val byteArray = serializer.toBytes(a) Buf.ByteArray.Owned(byteArray) } } abstract class ThriftOptionBijection[T <: ThriftStruct] extends Bijection[Buf, Option[T]] { val serializer: CompactThriftSerializer[T] override def apply(b: Buf): Option[T] = { if (b.isEmpty) { None } else { val byteArray = Buf.ByteArray.Owned.extract(b) Some(serializer.fromBytes(byteArray)) } } override def invert(a: Option[T]): Buf = { a match { case Some(t) => val byteArray = serializer.toBytes(t) Buf.ByteArray.Owned(byteArray) case None => Buf.Empty } } } class ThriftEnumBijection[T <: ThriftEnum](constructor: Int => T) extends Bijection[Buf, T] { override def apply(b: Buf): T = { val byteArray = Buf.ByteArray.Owned.extract(b) val byteBuffer = ByteBuffer.wrap(byteArray) constructor(byteBuffer.getInt()) } override def invert(a: T): Buf = { val byteBuffer: ByteBuffer = ByteBuffer.allocate(4) byteBuffer.putInt(a.getValue) Buf.ByteArray.Owned(byteBuffer.array()) } } class ThriftEnumOptionBijection[T <: ThriftEnum](constructor: Int => T) extends Bijection[Buf, Option[T]] { override def apply(b: Buf): Option[T] = { if (b.isEmpty) { None } else { val byteArray = Buf.ByteArray.Owned.extract(b) val byteBuffer = ByteBuffer.wrap(byteArray) Some(constructor(byteBuffer.getInt())) } } override def invert(a: Option[T]): Buf = { a match { case Some(obj) => { val byteBuffer: ByteBuffer = ByteBuffer.allocate(4) byteBuffer.putInt(obj.getValue) Buf.ByteArray.Owned(byteBuffer.array()) } case None => Buf.Empty } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-thrift-client", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common/BaseClientModule.scala ================================================ package com.twitter.follow_recommendations.common.clients.common import com.twitter.finagle.ThriftMux import com.twitter.finagle.thrift.Protocols import com.twitter.follow_recommendations.common.constants.ServiceConstants._ import com.twitter.inject.thrift.modules.ThriftClientModule import scala.reflect.ClassTag /** * basic client configurations that we apply for all of our clients go in here */ abstract class BaseClientModule[T: ClassTag] extends ThriftClientModule[T] { def configureThriftMuxClient(client: ThriftMux.Client): ThriftMux.Client = { client .withProtocolFactory( Protocols.binaryFactory( stringLengthLimit = StringLengthLimit, containerLengthLimit = ContainerLengthLimit)) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "cortex-deepbird/prediction/src/main/scala/com/twitter/cortex/deepbird/prediction", "cortex-deepbird/thrift/src/main/thrift:thrift-java", "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-core/src/main/scala", "finatra/inject/inject-thrift-client/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", "src/scala/com/twitter/ml/api/util", "util/util-core:scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/DeepBirdV2PredictionServiceClientModule.scala ================================================ package com.twitter.follow_recommendations.common.clients.deepbirdv2 import com.google.inject.Provides import com.google.inject.name.Named import com.twitter.bijection.scrooge.TBinaryProtocol import com.twitter.conversions.DurationOps._ import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService import com.twitter.finagle.ThriftMux import com.twitter.finagle.builder.ClientBuilder import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.mtls.client.MtlsStackClient._ import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.thrift.ClientId import com.twitter.finagle.thrift.RichClientParam import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.inject.TwitterModule /** * Module that provides multiple deepbirdv2 prediction service clients * We use the java api since data records are native java objects and we want to reduce overhead * while serializing/deserializing data. */ object DeepBirdV2PredictionServiceClientModule extends TwitterModule { val RequestTimeout = 300.millis private def getDeepbirdPredictionServiceClient( clientId: ClientId, label: String, dest: String, statsReceiver: StatsReceiver, serviceIdentifier: ServiceIdentifier ): DeepbirdPredictionService.ServiceToClient = { val clientStatsReceiver = statsReceiver.scope("clnt") val mTlsClient = ThriftMux.client.withClientId(clientId).withMutualTls(serviceIdentifier) new DeepbirdPredictionService.ServiceToClient( ClientBuilder() .name(label) .stack(mTlsClient) .dest(dest) .requestTimeout(RequestTimeout) .reportHostStats(NullStatsReceiver) .build(), RichClientParam( new TBinaryProtocol.Factory(), clientStats = clientStatsReceiver ) ) } @Provides @Named(GuiceNamedConstants.WTF_PROD_DEEPBIRDV2_CLIENT) def providesWtfProdDeepbirdV2PredictionService( clientId: ClientId, statsReceiver: StatsReceiver, serviceIdentifier: ServiceIdentifier ): DeepbirdPredictionService.ServiceToClient = { getDeepbirdPredictionServiceClient( clientId = clientId, label = "WtfProdDeepbirdV2PredictionService", dest = "/s/cassowary/deepbirdv2-hermit-wtf", statsReceiver = statsReceiver, serviceIdentifier = serviceIdentifier ) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/github/nscala_time", "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "src/thrift/com/twitter/onboarding/relevance/store:store-scala", "util/util-core:scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/DismissStore.scala ================================================ package com.twitter.follow_recommendations.common.clients.dismiss_store import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.finagle.stats.StatsReceiver import com.twitter.onboarding.relevance.store.thriftscala.WhoToFollowDismissEventDetails import com.twitter.stitch.Stitch import com.twitter.strato.catalog.Scan.Slice import com.twitter.strato.client.Scanner import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton /** * this store gets the list of dismissed candidates since a certain time * primarily used for filtering out accounts that a user has explicitly dismissed * * we fail open on timeouts, but loudly on other errors */ @Singleton class DismissStore @Inject() ( @Named(GuiceNamedConstants.DISMISS_STORE_SCANNER) scanner: Scanner[(Long, Slice[ (Long, Long) ]), Unit, (Long, (Long, Long)), WhoToFollowDismissEventDetails], stats: StatsReceiver) extends Logging { private val MaxCandidatesToReturn = 100 // gets a list of dismissed candidates. if numCandidatesToFetchOption is none, we will fetch the default number of candidates def get( userId: Long, negStartTimeMs: Long, maxCandidatesToFetchOption: Option[Int] ): Stitch[Seq[Long]] = { val maxCandidatesToFetch = maxCandidatesToFetchOption.getOrElse(MaxCandidatesToReturn) scanner .scan( ( userId, Slice( from = None, to = Some((negStartTimeMs, Long.MaxValue)), limit = Some(maxCandidatesToFetch) ) ) ) .map { case s: Seq[((Long, (Long, Long)), WhoToFollowDismissEventDetails)] if s.nonEmpty => s.map { case ((_: Long, (_: Long, candidateId: Long)), _: WhoToFollowDismissEventDetails) => candidateId } case _ => Nil } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "emailstorage/server/src/main/thrift/com/twitter/emailstorage/api:email-storage-service-scala", "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-thrift-client", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "stitch/stitch-core", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceClient.scala ================================================ package com.twitter.follow_recommendations.common.clients.email_storage_service import com.twitter.cds.contact_consent_state.thriftscala.PurposeOfProcessing import com.twitter.emailstorage.api.thriftscala.EmailStorageService import com.twitter.emailstorage.api.thriftscala.GetUsersEmailsRequest import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class EmailStorageServiceClient @Inject() ( val emailStorageService: EmailStorageService.MethodPerEndpoint) { def getVerifiedEmail( userId: Long, purposeOfProcessing: PurposeOfProcessing ): Stitch[Option[String]] = { val req = GetUsersEmailsRequest( userIds = Seq(userId), clientIdentifier = Some("follow-recommendations-service"), purposesOfProcessing = Some(Seq(purposeOfProcessing)) ) Stitch.callFuture(emailStorageService.getUsersEmails(req)) map { _.usersEmails.map(_.confirmedEmail.map(_.email)).head } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceModule.scala ================================================ package com.twitter.follow_recommendations.common.clients.email_storage_service import com.twitter.emailstorage.api.thriftscala.EmailStorageService import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.follow_recommendations.common.clients.common.BaseClientModule object EmailStorageServiceModule extends BaseClientModule[EmailStorageService.MethodPerEndpoint] with MtlsClient { override val label = "email-storage-service" override val dest = "/s/email-server/email-server" } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/github/nscala_time", "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-core/src/main/scala", "finatra/inject/inject-thrift-client/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "src/thrift/com/twitter/geoduck:geoduck-scala", "src/thrift/com/twitter/geoduck:geoduckpartnerplaces-thrift-scala", "stitch/stitch-core", "util/util-core:scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceClient.scala ================================================ package com.twitter.follow_recommendations.common.clients.geoduck import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode import com.twitter.geoduck.common.thriftscala.LocationSource import com.twitter.geoduck.common.thriftscala.PlaceQuery import com.twitter.geoduck.common.thriftscala.TransactionLocation import com.twitter.geoduck.common.thriftscala.UserLocationRequest import com.twitter.geoduck.thriftscala.LocationService import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class LocationServiceClient @Inject() (locationService: LocationService.MethodPerEndpoint) { def getGeohashAndCountryCode(userId: Long): Stitch[GeohashAndCountryCode] = { Stitch .callFuture { locationService .userLocation( UserLocationRequest( Seq(userId), Some(PlaceQuery(allPlaceTypes = Some(true))), simpleReverseGeocode = true)) .map(_.found.get(userId)).map { transactionLocationOpt => val geohashOpt = transactionLocationOpt.flatMap(getGeohashFromTransactionLocation) val countryCodeOpt = transactionLocationOpt.flatMap(_.simpleRgcResult.flatMap(_.countryCodeAlpha2)) GeohashAndCountryCode(geohashOpt, countryCodeOpt) } } } private[this] def getGeohashFromTransactionLocation( transactionLocation: TransactionLocation ): Option[String] = { transactionLocation.geohash.flatMap { geohash => val geohashPrefixLength = transactionLocation.locationSource match { // if location source is logical, keep the first 4 chars in geohash case Some(LocationSource.Logical) => Some(4) // if location source is physical, keep the prefix according to accuracy // accuracy is the accuracy of GPS readings in the unit of meter case Some(LocationSource.Physical) => transactionLocation.coordinate.flatMap { coordinate => coordinate.accuracy match { case Some(accuracy) if (accuracy < 50) => Some(7) case Some(accuracy) if (accuracy < 200) => Some(6) case Some(accuracy) if (accuracy < 1000) => Some(5) case Some(accuracy) if (accuracy < 50000) => Some(4) case Some(accuracy) if (accuracy < 100000) => Some(3) case _ => None } } case Some(LocationSource.Model) => Some(4) case _ => None } geohashPrefixLength match { case Some(l: Int) => geohash.stringGeohash.map(_.take(l)) case _ => None } } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceModule.scala ================================================ package com.twitter.follow_recommendations.common.clients.geoduck import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.follow_recommendations.common.clients.common.BaseClientModule import com.twitter.geoduck.thriftscala.LocationService object LocationServiceModule extends BaseClientModule[LocationService.MethodPerEndpoint] with MtlsClient { override val label = "geoduck_locationservice" override val dest = "/s/geo/geoduck_locationservice" } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/ReverseGeocodeClient.scala ================================================ package com.twitter.follow_recommendations.common.clients.geoduck import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode import com.twitter.geoduck.common.thriftscala.Location import com.twitter.geoduck.common.thriftscala.PlaceQuery import com.twitter.geoduck.common.thriftscala.ReverseGeocodeIPRequest import com.twitter.geoduck.service.thriftscala.GeoContext import com.twitter.geoduck.thriftscala.ReverseGeocoder import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class ReverseGeocodeClient @Inject() (rgcService: ReverseGeocoder.MethodPerEndpoint) { def getGeohashAndCountryCode(ipAddress: String): Stitch[GeohashAndCountryCode] = { Stitch .callFuture { rgcService .reverseGeocodeIp( ReverseGeocodeIPRequest( Seq(ipAddress), PlaceQuery(None), simpleReverseGeocode = true ) // note: simpleReverseGeocode means that country code will be included in response ).map { response => response.found.get(ipAddress) match { case Some(location) => getGeohashAndCountryCodeFromLocation(location) case _ => GeohashAndCountryCode(None, None) } } } } private def getGeohashAndCountryCodeFromLocation(location: Location): GeohashAndCountryCode = { val countryCode: Option[String] = location.simpleRgcResult.flatMap { _.countryCodeAlpha2 } val geohashString: Option[String] = location.geohash.flatMap { hash => hash.stringGeohash.flatMap { hashString => Some(ReverseGeocodeClient.truncate(hashString)) } } GeohashAndCountryCode(geohashString, countryCode) } } object ReverseGeocodeClient { val DefaultGeoduckIPRequestContext: GeoContext = GeoContext(allPlaceTypes = true, includeGeohash = true, includeCountryCode = true) // All these geohashes are guessed by IP (Logical Location Source). // So take the four letters to make sure it is consistent with LocationServiceClient val GeohashLengthAfterTruncation = 4 def truncate(geohash: String): String = geohash.take(GeohashLengthAfterTruncation) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/UserLocationFetcher.scala ================================================ package com.twitter.follow_recommendations.common.clients.geoduck import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class UserLocationFetcher @Inject() ( locationServiceClient: LocationServiceClient, reverseGeocodeClient: ReverseGeocodeClient, statsReceiver: StatsReceiver) { private val stats: StatsReceiver = statsReceiver.scope("user_location_fetcher") private val totalRequestsCounter = stats.counter("requests") private val emptyResponsesCounter = stats.counter("empty") private val locationServiceExceptionCounter = stats.counter("location_service_exception") private val reverseGeocodeExceptionCounter = stats.counter("reverse_geocode_exception") def getGeohashAndCountryCode( userId: Option[Long], ipAddress: Option[String] ): Stitch[Option[GeohashAndCountryCode]] = { totalRequestsCounter.incr() val lscLocationStitch = Stitch .collect { userId.map(locationServiceClient.getGeohashAndCountryCode) }.rescue { case _: Exception => locationServiceExceptionCounter.incr() Stitch.None } val ipLocationStitch = Stitch .collect { ipAddress.map(reverseGeocodeClient.getGeohashAndCountryCode) }.rescue { case _: Exception => reverseGeocodeExceptionCounter.incr() Stitch.None } Stitch.join(lscLocationStitch, ipLocationStitch).map { case (lscLocation, ipLocation) => { val geohash = lscLocation.flatMap(_.geohash).orElse(ipLocation.flatMap(_.geohash)) val countryCode = lscLocation.flatMap(_.countryCode).orElse(ipLocation.flatMap(_.countryCode)) (geohash, countryCode) match { case (None, None) => emptyResponsesCounter.incr() None case _ => Some(GeohashAndCountryCode(geohash, countryCode)) } } } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/github/nscala_time", "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-core/src/main/scala", "finatra/inject/inject-thrift-client/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", "src/thrift/com/twitter/gizmoduck:thrift-scala", "stitch/stitch-gizmoduck", "util/util-core:scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckClient.scala ================================================ package com.twitter.follow_recommendations.common.clients.gizmoduck import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.StatsUtil import com.twitter.gizmoduck.thriftscala.LookupContext import com.twitter.gizmoduck.thriftscala.PerspectiveEdge import com.twitter.gizmoduck.thriftscala.QueryFields import com.twitter.stitch.Stitch import com.twitter.stitch.gizmoduck.Gizmoduck import javax.inject.Inject import javax.inject.Singleton @Singleton class GizmoduckClient @Inject() (gizmoduckStitchClient: Gizmoduck, statsReceiver: StatsReceiver) { val stats = statsReceiver.scope("gizmoduck_client") val getByIdStats = stats.scope("get_by_id") val getUserById = stats.scope("get_user_by_id") def isProtected(userId: Long): Stitch[Boolean] = { // get latency metrics with StatsUtil.profileStitch when calling .getById val response = StatsUtil.profileStitch( gizmoduckStitchClient.getById(userId, Set(QueryFields.Safety)), getByIdStats ) response.map { result => result.user.flatMap(_.safety).map(_.isProtected).getOrElse(true) } } def getUserName(userId: Long, forUserId: Long): Stitch[Option[String]] = { val queryFields = GizmoduckClient.GetUserByIdUserNameQueryFields val lookupContext = LookupContext( forUserId = Some(forUserId), perspectiveEdges = Some(GizmoduckClient.DefaultPerspectiveEdges) ) // get latency metrics with StatsUtil.profileStitch when calling .getUserById val response = StatsUtil.profileStitch( gizmoduckStitchClient.getUserById(userId, queryFields, lookupContext), getUserById ) response.map(_.profile.map(_.name)) } } object GizmoduckClient { // Similar to GizmoduckUserRepository.DefaultPerspectiveEdges val DefaultPerspectiveEdges: Set[PerspectiveEdge] = Set( PerspectiveEdge.Blocking, PerspectiveEdge.BlockedBy, PerspectiveEdge.DeviceFollowing, PerspectiveEdge.FollowRequestSent, PerspectiveEdge.Following, PerspectiveEdge.FollowedBy, PerspectiveEdge.LifelineFollowing, PerspectiveEdge.LifelineFollowedBy, PerspectiveEdge.Muting, PerspectiveEdge.NoRetweetsFrom ) // From GizmoduckUserRepository.DefaultQueryFields val GetUserByIdQueryFields: Set[QueryFields] = Set( QueryFields.Account, QueryFields.Counts, QueryFields.ExtendedProfile, QueryFields.Perspective, QueryFields.Profile, QueryFields.ProfileDesign, QueryFields.ProfileLocation, QueryFields.Safety, QueryFields.Roles, QueryFields.Takedowns, QueryFields.UrlEntities, QueryFields.DirectMessageView, QueryFields.MediaView ) val GetUserByIdUserNameQueryFields: Set[QueryFields] = Set( QueryFields.Profile ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckModule.scala ================================================ package com.twitter.follow_recommendations.common.clients.gizmoduck import com.google.inject.Provides import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.follow_recommendations.common.clients.common.BaseClientModule import com.twitter.gizmoduck.thriftscala.QueryFields import com.twitter.gizmoduck.thriftscala.UserService import com.twitter.stitch.gizmoduck.Gizmoduck import javax.inject.Singleton object GizmoduckModule extends BaseClientModule[UserService.MethodPerEndpoint] with MtlsClient { override val label = "gizmoduck" override val dest = "/s/gizmoduck/gizmoduck" @Provides @Singleton def provideExtraGizmoduckQueryFields: Set[QueryFields] = Set.empty @Provides @Singleton def providesStitchClient(futureIface: UserService.MethodPerEndpoint): Gizmoduck = { Gizmoduck(futureIface) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-thrift-client", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "graph-feature-service/src/main/thrift/com/twitter/graph_feature_service:graph_feature_service_thrift-scala", "stitch/stitch-core", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureServiceClient.scala ================================================ package com.twitter.follow_recommendations.common.clients.graph_feature_service import com.twitter.follow_recommendations.common.models.FollowProof import com.twitter.graph_feature_service.thriftscala.PresetFeatureTypes.WtfTwoHop import com.twitter.graph_feature_service.thriftscala.EdgeType import com.twitter.graph_feature_service.thriftscala.GfsIntersectionResponse import com.twitter.graph_feature_service.thriftscala.GfsPresetIntersectionRequest import com.twitter.graph_feature_service.thriftscala.{Server => GraphFeatureService} import com.twitter.stitch.Stitch import javax.inject.{Inject, Singleton} @Singleton class GraphFeatureServiceClient @Inject() ( graphFeatureService: GraphFeatureService.MethodPerEndpoint) { import GraphFeatureServiceClient._ def getIntersections( userId: Long, candidateIds: Seq[Long], numIntersectionIds: Int ): Stitch[Map[Long, FollowProof]] = { Stitch .callFuture( graphFeatureService.getPresetIntersection( GfsPresetIntersectionRequest(userId, candidateIds, WtfTwoHop, Some(numIntersectionIds)) ) ).map { case GfsIntersectionResponse(gfsIntersectionResults) => (for { candidateId <- candidateIds gfsIntersectionResultForCandidate = gfsIntersectionResults.filter(_.candidateUserId == candidateId) followProof <- for { result <- gfsIntersectionResultForCandidate intersection <- result.intersectionValues if leftEdgeTypes.contains(intersection.featureType.leftEdgeType) if rightEdgeTypes.contains(intersection.featureType.rightEdgeType) intersectionIds <- intersection.intersectionIds.toSeq } yield FollowProof(intersectionIds, intersection.count.getOrElse(0)) } yield { candidateId -> followProof }).toMap } } } object GraphFeatureServiceClient { val leftEdgeTypes: Set[EdgeType] = Set(EdgeType.Following) val rightEdgeTypes: Set[EdgeType] = Set(EdgeType.FollowedBy) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureStoreModule.scala ================================================ package com.twitter.follow_recommendations.common.clients.graph_feature_service import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.follow_recommendations.common.clients.common.BaseClientModule import com.twitter.graph_feature_service.thriftscala.{Server => GraphFeatureService} object GraphFeatureStoreModule extends BaseClientModule[GraphFeatureService.MethodPerEndpoint] with MtlsClient { override val label = "graph_feature_service" override val dest = "/s/cassowary/graph_feature_service-server" } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/github/nscala_time", "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "stitch/stitch-socialgraph", "util/util-core:scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/ImpressionStoreModule.scala ================================================ package com.twitter.follow_recommendations.common.clients.impression_store import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.follow_recommendations.thriftscala.DisplayLocation import com.twitter.inject.TwitterModule import com.twitter.strato.catalog.Scan.Slice import com.twitter.strato.client.Client import com.twitter.strato.thrift.ScroogeConvImplicits._ object ImpressionStoreModule extends TwitterModule { val columnPath: String = "onboarding/userrecs/wtfImpressionCountsStore" type PKey = (Long, DisplayLocation) type LKey = Long type Value = (Long, Int) @Provides @Singleton def providesImpressionStore(stratoClient: Client): WtfImpressionStore = { new WtfImpressionStore( stratoClient.scanner[ (PKey, Slice[LKey]), Unit, (PKey, LKey), Value ](columnPath) ) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/WtfImpressionStore.scala ================================================ package com.twitter.follow_recommendations.common.clients.impression_store import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.follow_recommendations.common.models.WtfImpression import com.twitter.follow_recommendations.thriftscala.{DisplayLocation => TDisplayLocation} import com.twitter.stitch.Stitch import com.twitter.strato.catalog.Scan.Slice import com.twitter.strato.client.Scanner import com.twitter.util.Time import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton @Singleton class WtfImpressionStore @Inject() ( scanner: Scanner[ ((Long, TDisplayLocation), Slice[Long]), Unit, ((Long, TDisplayLocation), Long), (Long, Int) ]) extends Logging { def get(userId: Long, dl: DisplayLocation): Stitch[Seq[WtfImpression]] = { val thriftDl = dl.toThrift scanner.scan(((userId, thriftDl), Slice.all[Long])).map { impressionsPerDl => val wtfImpressions = for { (((_, _), candidateId), (latestTs, counts)) <- impressionsPerDl } yield WtfImpression( candidateId = candidateId, displayLocation = dl, latestTime = Time.fromMilliseconds(latestTs), counts = counts ) wtfImpressions } rescue { // fail open so that the request can still go through case ex: Throwable => logger.warn(s"$dl WtfImpressionsStore warn: " + ex.getMessage) Stitch.Nil } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/BUILD ================================================ scala_library( name = "interests_service", sources = ["InterestServiceClient.scala"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/interests", "interests-service/thrift/src/main/thrift:thrift-scala", "strato/src/main/scala/com/twitter/strato/catalog", "strato/src/main/scala/com/twitter/strato/client", "strato/src/main/scala/com/twitter/strato/data", "strato/src/main/scala/com/twitter/strato/thrift", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/InterestServiceClient.scala ================================================ package com.twitter.follow_recommendations.common.clients.interests_service import com.google.inject.Inject import com.google.inject.Singleton import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.store.InterestedInInterestsFetchKey import com.twitter.inject.Logging import com.twitter.interests.thriftscala.InterestId import com.twitter.interests.thriftscala.InterestRelationship import com.twitter.interests.thriftscala.InterestedInInterestModel import com.twitter.interests.thriftscala.UserInterest import com.twitter.interests.thriftscala.UserInterestData import com.twitter.interests.thriftscala.UserInterestsResponse import com.twitter.stitch.Stitch import com.twitter.strato.client.Client import com.twitter.strato.thrift.ScroogeConvImplicits._ @Singleton class InterestServiceClient @Inject() ( stratoClient: Client, statsReceiver: StatsReceiver = NullStatsReceiver) extends Logging { val interestsServiceStratoColumnPath = "interests/interestedInInterests" val stats = statsReceiver.scope("interest_service_client") val errorCounter = stats.counter("error") private val interestsFetcher = stratoClient.fetcher[InterestedInInterestsFetchKey, UserInterestsResponse]( interestsServiceStratoColumnPath, checkTypes = true ) def fetchUttInterestIds( userId: Long ): Stitch[Seq[Long]] = { fetchInterestRelationships(userId) .map(_.toSeq.flatten.flatMap(extractUttInterest)) } def extractUttInterest( interestRelationShip: InterestRelationship ): Option[Long] = { interestRelationShip match { case InterestRelationship.V1(relationshipV1) => relationshipV1.interestId match { case InterestId.SemanticCore(semanticCoreInterest) => Some(semanticCoreInterest.id) case _ => None } case _ => None } } def fetchCustomInterests( userId: Long ): Stitch[Seq[String]] = { fetchInterestRelationships(userId) .map(_.toSeq.flatten.flatMap(extractCustomInterest)) } def extractCustomInterest( interestRelationShip: InterestRelationship ): Option[String] = { interestRelationShip match { case InterestRelationship.V1(relationshipV1) => relationshipV1.interestId match { case InterestId.FreeForm(freeFormInterest) => Some(freeFormInterest.interest) case _ => None } case _ => None } } def fetchInterestRelationships( userId: Long ): Stitch[Option[Seq[InterestRelationship]]] = { interestsFetcher .fetch( InterestedInInterestsFetchKey( userId = userId, labels = None, None )) .map(_.v) .map { case Some(response) => response.interests.interests.map { interests => interests.collect { case UserInterest(_, Some(interestData)) => getInterestRelationship(interestData) }.flatten } case _ => None } .rescue { case e: Throwable => // we are swallowing all errors logger.warn(s"interests could not be retrieved for user $userId due to ${e.getCause}") errorCounter.incr Stitch.None } } private def getInterestRelationship( interestData: UserInterestData ): Seq[InterestRelationship] = { interestData match { case UserInterestData.InterestedIn(interestModels) => interestModels.collect { case InterestedInInterestModel.ExplicitModel(model) => model } case _ => Nil } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-thrift-client", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "phonestorage/server/src/main/thrift/com/twitter/phonestorage/api:phone-storage-service-scala", "stitch/stitch-core", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceClient.scala ================================================ package com.twitter.follow_recommendations.common.clients.phone_storage_service import com.twitter.cds.contact_consent_state.thriftscala.PurposeOfProcessing import com.twitter.phonestorage.api.thriftscala.GetUserPhonesByUsersRequest import com.twitter.phonestorage.api.thriftscala.PhoneStorageService import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class PhoneStorageServiceClient @Inject() ( val phoneStorageService: PhoneStorageService.MethodPerEndpoint) { /** * PSS can potentially return multiple phone records. * The current implementation of getUserPhonesByUsers returns only a single phone for a single user_id but * we can trivially support handling multiple in case that changes in the future. */ def getPhoneNumbers( userId: Long, purposeOfProcessing: PurposeOfProcessing, forceCarrierLookup: Option[Boolean] = None ): Stitch[Seq[String]] = { val req = GetUserPhonesByUsersRequest( userIds = Seq(userId), forceCarrierLookup = forceCarrierLookup, purposesOfProcessing = Some(Seq(purposeOfProcessing)) ) Stitch.callFuture(phoneStorageService.getUserPhonesByUsers(req)) map { _.userPhones.map(_.phoneNumber) } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceModule.scala ================================================ package com.twitter.follow_recommendations.common.clients.phone_storage_service import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.follow_recommendations.common.clients.common.BaseClientModule import com.twitter.phonestorage.api.thriftscala.PhoneStorageService object PhoneStorageServiceModule extends BaseClientModule[PhoneStorageService.MethodPerEndpoint] with MtlsClient { override val label = "phone-storage-service" override val dest = "/s/ibis-ds-api/ibis-ds-api:thrift2" } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "strato/config/columns/ml/featureStore:featureStore-strato-client", "strato/config/columns/onboarding/userrecs:userrecs-strato-client", "strato/src/main/scala/com/twitter/strato/client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/Engagement.scala ================================================ package com.twitter.follow_recommendations.common.clients.real_time_real_graph sealed trait EngagementType // We do not include SoftFollow since it's deprecated object EngagementType { object Click extends EngagementType object Like extends EngagementType object Mention extends EngagementType object Retweet extends EngagementType object ProfileView extends EngagementType } case class Engagement(engagementType: EngagementType, timestamp: Long) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/EngagementScorer.scala ================================================ package com.twitter.follow_recommendations.common.clients.real_time_real_graph import com.twitter.conversions.DurationOps._ import com.twitter.util.Time object EngagementScorer { private[real_time_real_graph] val MemoryDecayHalfLife = 24.hour private val ScoringFunctionBase = 0.5 def apply( engagements: Map[Long, Seq[Engagement]], engagementScoreMap: Map[EngagementType, Double], minScore: Double = 0.0 ): Seq[(Long, Double, Seq[EngagementType])] = { val now = Time.now engagements .mapValues { engags => val totalScore = engags.map { engagement => score(engagement, now, engagementScoreMap) }.sum val engagementProof = getEngagementProof(engags, engagementScoreMap) (totalScore, engagementProof) } .collect { case (uid, (score, proof)) if score > minScore => (uid, score, proof) } .toSeq .sortBy(-_._2) } /** * The engagement score is the base score decayed via timestamp, loosely model the human memory forgetting * curve, see https://en.wikipedia.org/wiki/Forgetting_curve */ private[real_time_real_graph] def score( engagement: Engagement, now: Time, engagementScoreMap: Map[EngagementType, Double] ): Double = { val timeLapse = math.max(now.inMillis - engagement.timestamp, 0) val engagementScore = engagementScoreMap.getOrElse(engagement.engagementType, 0.0) engagementScore * math.pow( ScoringFunctionBase, timeLapse.toDouble / MemoryDecayHalfLife.inMillis) } private def getEngagementProof( engagements: Seq[Engagement], engagementScoreMap: Map[EngagementType, Double] ): Seq[EngagementType] = { val filteredEngagement = engagements .collectFirst { case engagement if engagement.engagementType != EngagementType.Click && engagementScoreMap.get(engagement.engagementType).exists(_ > 0.0) => engagement.engagementType } Seq(filteredEngagement.getOrElse(EngagementType.Click)) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/RealTimeRealGraphClient.scala ================================================ package com.twitter.follow_recommendations.common.clients.real_time_real_graph import com.google.inject.Inject import com.google.inject.Singleton import com.twitter.conversions.DurationOps._ import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.ml.featureStore.TimelinesUserVertexOnUserClientColumn import com.twitter.strato.generated.client.onboarding.userrecs.RealGraphScoresMhOnUserClientColumn import com.twitter.util.Duration import com.twitter.util.Time import com.twitter.wtf.real_time_interaction_graph.thriftscala._ @Singleton class RealTimeRealGraphClient @Inject() ( timelinesUserVertexOnUserClientColumn: TimelinesUserVertexOnUserClientColumn, realGraphScoresMhOnUserClientColumn: RealGraphScoresMhOnUserClientColumn) { def mapUserVertexToEngagementAndFilter(userVertex: UserVertex): Map[Long, Seq[Engagement]] = { val minTimestamp = (Time.now - RealTimeRealGraphClient.MaxEngagementAge).inMillis userVertex.outgoingInteractionMap.mapValues { interactions => interactions .flatMap { interaction => RealTimeRealGraphClient.toEngagement(interaction) }.filter( _.timestamp >= minTimestamp) }.toMap } def getRecentProfileViewEngagements(userId: Long): Stitch[Map[Long, Seq[Engagement]]] = { timelinesUserVertexOnUserClientColumn.fetcher .fetch(userId).map(_.v).map { input => input .map { userVertex => val targetToEngagements = mapUserVertexToEngagementAndFilter(userVertex) targetToEngagements.mapValues { engagements => engagements.filter(engagement => engagement.engagementType == EngagementType.ProfileView) } }.getOrElse(Map.empty) } } def getUsersRecentlyEngagedWith( userId: Long, engagementScoreMap: Map[EngagementType, Double], includeDirectFollowCandidates: Boolean, includeNonDirectFollowCandidates: Boolean ): Stitch[Seq[CandidateUser]] = { val isNewUser = SnowflakeId.timeFromIdOpt(userId).exists { signupTime => (Time.now - signupTime) < RealTimeRealGraphClient.MaxNewUserAge } val updatedEngagementScoreMap = if (isNewUser) engagementScoreMap + (EngagementType.ProfileView -> RealTimeRealGraphClient.ProfileViewScore) else engagementScoreMap Stitch .join( timelinesUserVertexOnUserClientColumn.fetcher.fetch(userId).map(_.v), realGraphScoresMhOnUserClientColumn.fetcher.fetch(userId).map(_.v)).map { case (Some(userVertex), Some(neighbors)) => val engagements = mapUserVertexToEngagementAndFilter(userVertex) val candidatesAndScores: Seq[(Long, Double, Seq[EngagementType])] = EngagementScorer.apply(engagements, engagementScoreMap = updatedEngagementScoreMap) val directNeighbors = neighbors.candidates.map(_._1).toSet val (directFollows, nonDirectFollows) = candidatesAndScores .partition { case (id, _, _) => directNeighbors.contains(id) } val candidates = (if (includeNonDirectFollowCandidates) nonDirectFollows else Seq.empty) ++ (if (includeDirectFollowCandidates) directFollows.take(RealTimeRealGraphClient.MaxNumDirectFollow) else Seq.empty) candidates.map { case (id, score, proof) => CandidateUser(id, Some(score)) } case _ => Nil } } def getRealGraphWeights(userId: Long): Stitch[Map[Long, Double]] = realGraphScoresMhOnUserClientColumn.fetcher .fetch(userId) .map( _.v .map(_.candidates.map(candidate => (candidate.userId, candidate.score)).toMap) .getOrElse(Map.empty[Long, Double])) } object RealTimeRealGraphClient { private def toEngagement(interaction: Interaction): Option[Engagement] = { // We do not include SoftFollow since it's deprecated interaction match { case Interaction.Retweet(Retweet(timestamp)) => Some(Engagement(EngagementType.Retweet, timestamp)) case Interaction.Favorite(Favorite(timestamp)) => Some(Engagement(EngagementType.Like, timestamp)) case Interaction.Click(Click(timestamp)) => Some(Engagement(EngagementType.Click, timestamp)) case Interaction.Mention(Mention(timestamp)) => Some(Engagement(EngagementType.Mention, timestamp)) case Interaction.ProfileView(ProfileView(timestamp)) => Some(Engagement(EngagementType.ProfileView, timestamp)) case _ => None } } val MaxNumDirectFollow = 50 val MaxEngagementAge: Duration = 14.days val MaxNewUserAge: Duration = 30.days val ProfileViewScore = 0.4 val EngagementScoreMap = Map( EngagementType.Like -> 1.0, EngagementType.Retweet -> 1.0, EngagementType.Mention -> 1.0 ) val StrongEngagementScoreMap = Map( EngagementType.Like -> 1.0, EngagementType.Retweet -> 1.0, ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/github/nscala_time", "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-core/src/main/scala", "finatra/inject/inject-thrift-client/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "socialgraph/server/src/main/scala/com/twitter/socialgraph/util", "src/thrift/com/twitter/socialgraph:thrift-scala", "stitch/stitch-socialgraph", "strato/config/columns/onboarding/socialGraphService:socialGraphService-strato-client", "strato/src/main/scala/com/twitter/strato/client", "util/util-core:scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphClient.scala ================================================ package com.twitter.follow_recommendations.common.clients.socialgraph import com.twitter.escherbird.util.stitchcache.StitchCache import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.StatsUtil import com.twitter.follow_recommendations.common.models.FollowProof import com.twitter.follow_recommendations.common.models.UserIdWithTimestamp import com.twitter.inject.Logging import com.twitter.socialgraph.thriftscala.EdgesRequest import com.twitter.socialgraph.thriftscala.IdsRequest import com.twitter.socialgraph.thriftscala.IdsResult import com.twitter.socialgraph.thriftscala.LookupContext import com.twitter.socialgraph.thriftscala.OverCapacity import com.twitter.socialgraph.thriftscala.PageRequest import com.twitter.socialgraph.thriftscala.RelationshipType import com.twitter.socialgraph.thriftscala.SrcRelationship import com.twitter.socialgraph.util.ByteBufferUtil import com.twitter.stitch.Stitch import com.twitter.stitch.socialgraph.SocialGraph import com.twitter.strato.client.Fetcher import com.twitter.strato.generated.client.onboarding.socialGraphService.IdsClientColumn import com.twitter.util.Duration import com.twitter.util.Time import java.nio.ByteBuffer import javax.inject.Inject import javax.inject.Singleton case class RecentEdgesQuery( userId: Long, relations: Seq[RelationshipType], // prefer to default value to better utilize the caching function of stitch count: Option[Int] = Some(SocialGraphClient.MaxQuerySize), performUnion: Boolean = true, recentEdgesWindowOpt: Option[Duration] = None, targets: Option[Seq[Long]] = None) case class EdgeRequestQuery( userId: Long, relation: RelationshipType, count: Option[Int] = Some(SocialGraphClient.MaxQuerySize), performUnion: Boolean = true, recentEdgesWindowOpt: Option[Duration] = None, targets: Option[Seq[Long]] = None) @Singleton class SocialGraphClient @Inject() ( socialGraph: SocialGraph, idsClientColumn: IdsClientColumn, statsReceiver: StatsReceiver = NullStatsReceiver) extends Logging { private val stats = statsReceiver.scope(this.getClass.getSimpleName) private val cacheStats = stats.scope("cache") private val getIntersectionsStats = stats.scope("getIntersections") private val getIntersectionsFromCachedColumnStats = stats.scope("getIntersectionsFromCachedColumn") private val getRecentEdgesStats = stats.scope("getRecentEdges") private val getRecentEdgesCachedStats = stats.scope("getRecentEdgesCached") private val getRecentEdgesFromCachedColumnStats = stats.scope("getRecentEdgesFromCachedColumn") private val getRecentEdgesCachedInternalStats = stats.scope("getRecentEdgesCachedInternal") private val getRecentEdgesWithTimeStats = stats.scope("getRecentEdgesWithTime") val sgsIdsFetcher: Fetcher[IdsRequest, Unit, IdsResult] = idsClientColumn.fetcher private val recentEdgesCache = StitchCache[RecentEdgesQuery, Seq[Long]]( maxCacheSize = SocialGraphClient.MaxCacheSize, ttl = SocialGraphClient.CacheTTL, statsReceiver = cacheStats, underlyingCall = getRecentEdges ) def getRecentEdgesCached( rq: RecentEdgesQuery, useCachedStratoColumn: Boolean = true ): Stitch[Seq[Long]] = { getRecentEdgesCachedStats.counter("requests").incr() if (useCachedStratoColumn) { getRecentEdgesFromCachedColumn(rq) } else { StatsUtil.profileStitch( getRecentEdgesCachedInternal(rq), getRecentEdgesCachedInternalStats ) } } def getRecentEdgesCachedInternal(rq: RecentEdgesQuery): Stitch[Seq[Long]] = { recentEdgesCache.readThrough(rq) } def getRecentEdgesFromCachedColumn(rq: RecentEdgesQuery): Stitch[Seq[Long]] = { val pageRequest = rq.recentEdgesWindowOpt match { case Some(recentEdgesWindow) => PageRequest( count = rq.count, cursor = Some(getEdgeCursor(recentEdgesWindow)), selectAll = Some(true) ) case _ => PageRequest(count = rq.count) } val idsRequest = IdsRequest( rq.relations.map { relationshipType => SrcRelationship( source = rq.userId, relationshipType = relationshipType, targets = rq.targets ) }, pageRequest = Some(pageRequest), context = Some(LookupContext(performUnion = Some(rq.performUnion))) ) val socialGraphStitch = sgsIdsFetcher .fetch(idsRequest, Unit) .map(_.v) .map { result => result .map { idResult => val userIds: Seq[Long] = idResult.ids getRecentEdgesFromCachedColumnStats.stat("num_edges").add(userIds.size) userIds }.getOrElse(Seq.empty) } .rescue { case e: Exception => stats.counter(e.getClass.getSimpleName).incr() Stitch.Nil } StatsUtil.profileStitch( socialGraphStitch, getRecentEdgesFromCachedColumnStats ) } def getRecentEdges(rq: RecentEdgesQuery): Stitch[Seq[Long]] = { val pageRequest = rq.recentEdgesWindowOpt match { case Some(recentEdgesWindow) => PageRequest( count = rq.count, cursor = Some(getEdgeCursor(recentEdgesWindow)), selectAll = Some(true) ) case _ => PageRequest(count = rq.count) } val socialGraphStitch = socialGraph .ids( IdsRequest( rq.relations.map { relationshipType => SrcRelationship( source = rq.userId, relationshipType = relationshipType, targets = rq.targets ) }, pageRequest = Some(pageRequest), context = Some(LookupContext(performUnion = Some(rq.performUnion))) ) ) .map { idsResult => val userIds: Seq[Long] = idsResult.ids getRecentEdgesStats.stat("num_edges").add(userIds.size) userIds } .rescue { case e: OverCapacity => stats.counter(e.getClass.getSimpleName).incr() logger.warn("SGS Over Capacity", e) Stitch.Nil } StatsUtil.profileStitch( socialGraphStitch, getRecentEdgesStats ) } // This method return recent edges of (userId, timeInMs) def getRecentEdgesWithTime(rq: EdgeRequestQuery): Stitch[Seq[UserIdWithTimestamp]] = { val pageRequest = rq.recentEdgesWindowOpt match { case Some(recentEdgesWindow) => PageRequest( count = rq.count, cursor = Some(getEdgeCursor(recentEdgesWindow)), selectAll = Some(true) ) case _ => PageRequest(count = rq.count) } val socialGraphStitch = socialGraph .edges( EdgesRequest( SrcRelationship( source = rq.userId, relationshipType = rq.relation, targets = rq.targets ), pageRequest = Some(pageRequest), context = Some(LookupContext(performUnion = Some(rq.performUnion))) ) ) .map { edgesResult => val userIds = edgesResult.edges.map { socialEdge => UserIdWithTimestamp(socialEdge.target, socialEdge.updatedAt) } getRecentEdgesWithTimeStats.stat("num_edges").add(userIds.size) userIds } .rescue { case e: OverCapacity => stats.counter(e.getClass.getSimpleName).incr() logger.warn("SGS Over Capacity", e) Stitch.Nil } StatsUtil.profileStitch( socialGraphStitch, getRecentEdgesWithTimeStats ) } // This method returns the cursor for a time duration, such that all the edges returned by SGS will be created // in the range (now-window, now) def getEdgeCursor(window: Duration): ByteBuffer = { val cursorInLong = (-(Time.now - window).inMilliseconds) << 20 ByteBufferUtil.fromLong(cursorInLong) } // notice that this is more expensive but more realtime than the GFS one def getIntersections( userId: Long, candidateIds: Seq[Long], numIntersectionIds: Int ): Stitch[Map[Long, FollowProof]] = { val socialGraphStitch: Stitch[Map[Long, FollowProof]] = Stitch .collect(candidateIds.map { candidateId => socialGraph .ids( IdsRequest( Seq( SrcRelationship(userId, RelationshipType.Following), SrcRelationship(candidateId, RelationshipType.FollowedBy) ), pageRequest = Some(PageRequest(count = Some(numIntersectionIds))) ) ).map { idsResult => getIntersectionsStats.stat("num_edges").add(idsResult.ids.size) (candidateId -> FollowProof(idsResult.ids, idsResult.ids.size)) } }).map(_.toMap) .rescue { case e: OverCapacity => stats.counter(e.getClass.getSimpleName).incr() logger.warn("social graph over capacity in hydrating social proof", e) Stitch.value(Map.empty) } StatsUtil.profileStitch( socialGraphStitch, getIntersectionsStats ) } def getIntersectionsFromCachedColumn( userId: Long, candidateIds: Seq[Long], numIntersectionIds: Int ): Stitch[Map[Long, FollowProof]] = { val socialGraphStitch: Stitch[Map[Long, FollowProof]] = Stitch .collect(candidateIds.map { candidateId => val idsRequest = IdsRequest( Seq( SrcRelationship(userId, RelationshipType.Following), SrcRelationship(candidateId, RelationshipType.FollowedBy) ), pageRequest = Some(PageRequest(count = Some(numIntersectionIds))) ) sgsIdsFetcher .fetch(idsRequest, Unit) .map(_.v) .map { resultOpt => resultOpt.map { idsResult => getIntersectionsFromCachedColumnStats.stat("num_edges").add(idsResult.ids.size) candidateId -> FollowProof(idsResult.ids, idsResult.ids.size) } } }).map(_.flatten.toMap) .rescue { case e: Exception => stats.counter(e.getClass.getSimpleName).incr() Stitch.value(Map.empty) } StatsUtil.profileStitch( socialGraphStitch, getIntersectionsFromCachedColumnStats ) } def getInvalidRelationshipUserIds( userId: Long, maxNumRelationship: Int = SocialGraphClient.MaxNumInvalidRelationship ): Stitch[Seq[Long]] = { getRecentEdges( RecentEdgesQuery( userId, SocialGraphClient.InvalidRelationshipTypes, Some(maxNumRelationship) ) ) } def getInvalidRelationshipUserIdsFromCachedColumn( userId: Long, maxNumRelationship: Int = SocialGraphClient.MaxNumInvalidRelationship ): Stitch[Seq[Long]] = { getRecentEdgesFromCachedColumn( RecentEdgesQuery( userId, SocialGraphClient.InvalidRelationshipTypes, Some(maxNumRelationship) ) ) } def getRecentFollowedUserIds(userId: Long): Stitch[Seq[Long]] = { getRecentEdges( RecentEdgesQuery( userId, Seq(RelationshipType.Following) ) ) } def getRecentFollowedUserIdsFromCachedColumn(userId: Long): Stitch[Seq[Long]] = { getRecentEdgesFromCachedColumn( RecentEdgesQuery( userId, Seq(RelationshipType.Following) ) ) } def getRecentFollowedUserIdsWithTime(userId: Long): Stitch[Seq[UserIdWithTimestamp]] = { getRecentEdgesWithTime( EdgeRequestQuery( userId, RelationshipType.Following ) ) } def getRecentFollowedByUserIds(userId: Long): Stitch[Seq[Long]] = { getRecentEdges( RecentEdgesQuery( userId, Seq(RelationshipType.FollowedBy) ) ) } def getRecentFollowedByUserIdsFromCachedColumn(userId: Long): Stitch[Seq[Long]] = { getRecentEdgesFromCachedColumn( RecentEdgesQuery( userId, Seq(RelationshipType.FollowedBy) ) ) } def getRecentFollowedUserIdsWithTimeWindow( userId: Long, timeWindow: Duration ): Stitch[Seq[Long]] = { getRecentEdges( RecentEdgesQuery( userId, Seq(RelationshipType.Following), recentEdgesWindowOpt = Some(timeWindow) ) ) } } object SocialGraphClient { val MaxQuerySize: Int = 500 val MaxCacheSize: Int = 5000000 // Ref: src/thrift/com/twitter/socialgraph/social_graph_service.thrift val MaxNumInvalidRelationship: Int = 5000 val CacheTTL: Duration = Duration.fromHours(24) val InvalidRelationshipTypes: Seq[RelationshipType] = Seq( RelationshipType.HideRecommendations, RelationshipType.Blocking, RelationshipType.BlockedBy, RelationshipType.Muting, RelationshipType.MutedBy, RelationshipType.ReportedAsSpam, RelationshipType.ReportedAsSpamBy, RelationshipType.ReportedAsAbuse, RelationshipType.ReportedAsAbuseBy, RelationshipType.FollowRequestOutgoing, RelationshipType.Following, RelationshipType.UsedToFollow, ) /** * * Whether to call SGS to validate each candidate based on the number of invalid relationship users * prefetched during request building step. This aims to not omit any invalid candidates that are * not filtered out in previous steps. * If the number is 0, this might be a fail-opened SGS call. * If the number is larger or equal to 5000, this could hit SGS page size limit. * Both cases account for a small percentage of the total traffic (<5%). * * @param numInvalidRelationshipUsers number of invalid relationship users fetched from getInvalidRelationshipUserIds * @return whether to enable post-ranker SGS predicate */ def enablePostRankerSgsPredicate(numInvalidRelationshipUsers: Int): Boolean = { numInvalidRelationshipUsers == 0 || numInvalidRelationshipUsers >= MaxNumInvalidRelationship } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphModule.scala ================================================ package com.twitter.follow_recommendations.common.clients.socialgraph import com.google.inject.Provides import com.twitter.finagle.ThriftMux import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.follow_recommendations.common.clients.common.BaseClientModule import com.twitter.socialgraph.thriftscala.SocialGraphService import com.twitter.stitch.socialgraph.SocialGraph import javax.inject.Singleton object SocialGraphModule extends BaseClientModule[SocialGraphService.MethodPerEndpoint] with MtlsClient { override val label = "social-graph-service" override val dest = "/s/socialgraph/socialgraph" override def configureThriftMuxClient(client: ThriftMux.Client): ThriftMux.Client = client.withSessionQualifier.noFailFast @Provides @Singleton def providesStitchClient(futureIface: SocialGraphService.MethodPerEndpoint): SocialGraph = { SocialGraph(futureIface) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "src/scala/com/twitter/onboarding/relevance/candidate_generation/utt/models", "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", "src/thrift/com/twitter/frigate/data_pipeline:frigate-user-history-thrift-scala", "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", "src/thrift/com/twitter/hermit/pop_geo:hermit-pop-geo-scala", "src/thrift/com/twitter/onboarding/relevance/relatable_accounts:relatable_accounts-scala", "src/thrift/com/twitter/onboarding/relevance/store:store-scala", "src/thrift/com/twitter/recos/user_user_graph:user_user_graph-scala", "src/thrift/com/twitter/search/account_search/extended_network:extended_network_users-scala", "src/thrift/com/twitter/service/metastore/gen:thrift-scala", "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", "src/thrift/com/twitter/wtf/ml:wtf-ml-output-thrift-scala", "src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-scala", "src/thrift/com/twitter/wtf/triangular_loop:triangular_loop-scala", "strato/src/main/scala/com/twitter/strato/client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/StratoClientModule.scala ================================================ package com.twitter.follow_recommendations.common.clients.strato import com.google.inject.name.Named import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.conversions.DurationOps._ import com.twitter.core_workflows.user_model.thriftscala.CondensedUserState import com.twitter.search.account_search.extended_network.thriftscala.ExtendedNetworkUserKey import com.twitter.search.account_search.extended_network.thriftscala.ExtendedNetworkUserVal import com.twitter.finagle.ThriftMux import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.thrift.Protocols import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.follow_recommendations.common.constants.ServiceConstants._ import com.twitter.frigate.data_pipeline.candidate_generation.thriftscala.LatestEvents import com.twitter.hermit.candidate.thriftscala.{Candidates => HermitCandidates} import com.twitter.hermit.pop_geo.thriftscala.PopUsersInPlace import com.twitter.onboarding.relevance.relatable_accounts.thriftscala.RelatableAccounts import com.twitter.inject.TwitterModule import com.twitter.onboarding.relevance.candidates.thriftscala.InterestBasedUserRecommendations import com.twitter.onboarding.relevance.candidates.thriftscala.UTTInterest import com.twitter.onboarding.relevance.store.thriftscala.WhoToFollowDismissEventDetails import com.twitter.recos.user_user_graph.thriftscala.RecommendUserRequest import com.twitter.recos.user_user_graph.thriftscala.RecommendUserResponse import com.twitter.service.metastore.gen.thriftscala.UserRecommendabilityFeatures import com.twitter.strato.catalog.Scan.Slice import com.twitter.strato.client.Strato.{Client => StratoClient} import com.twitter.strato.client.Client import com.twitter.strato.client.Fetcher import com.twitter.strato.client.Scanner import com.twitter.strato.thrift.ScroogeConvImplicits._ import com.twitter.wtf.candidate.thriftscala.CandidateSeq import com.twitter.wtf.ml.thriftscala.CandidateFeatures import com.twitter.wtf.real_time_interaction_graph.thriftscala.Interaction import com.twitter.wtf.triangular_loop.thriftscala.{Candidates => TriangularLoopCandidates} import com.twitter.strato.opcontext.Attribution._ object StratoClientModule extends TwitterModule { // column paths val CosineFollowPath = "recommendations/similarity/similarUsersByFollowGraph.User" val CosineListPath = "recommendations/similarity/similarUsersByListGraph.User" val CuratedCandidatesPath = "onboarding/curatedAccounts" val CuratedFilteredAccountsPath = "onboarding/filteredAccountsFromRecommendations" val PopUsersInPlacePath = "onboarding/userrecs/popUsersInPlace" val ProfileSidebarBlacklistPath = "recommendations/hermit/profile-sidebar-blacklist" val RealTimeInteractionsPath = "hmli/realTimeInteractions" val SimsPath = "recommendations/similarity/similarUsersBySims.User" val DBV2SimsPath = "onboarding/userrecs/newSims.User" val TriangularLoopsPath = "onboarding/userrecs/triangularLoops.User" val TwoHopRandomWalkPath = "onboarding/userrecs/twoHopRandomWalk.User" val UserRecommendabilityPath = "onboarding/userRecommendabilityWithLongKeys.User" val UTTAccountRecommendationsPath = "onboarding/userrecs/utt_account_recommendations" val UttSeedAccountsRecommendationPath = "onboarding/userrecs/utt_seed_accounts" val UserStatePath = "onboarding/userState.User" val WTFPostNuxFeaturesPath = "ml/featureStore/onboarding/wtfPostNuxFeatures.User" val ElectionCandidatesPath = "onboarding/electionAccounts" val UserUserGraphPath = "recommendations/userUserGraph" val WtfDissmissEventsPath = "onboarding/wtfDismissEvents" val RelatableAccountsPath = "onboarding/userrecs/relatableAccounts" val ExtendedNetworkCandidatesPath = "search/account_search/extendedNetworkCandidatesMH" val LabeledNotificationPath = "frigate/magicrecs/labeledPushRecsAggregated.User" @Provides @Singleton def stratoClient(serviceIdentifier: ServiceIdentifier): Client = { val timeoutBudget = 500.milliseconds StratoClient( ThriftMux.client .withRequestTimeout(timeoutBudget) .withProtocolFactory(Protocols.binaryFactory( stringLengthLimit = StringLengthLimit, containerLengthLimit = ContainerLengthLimit))) .withMutualTls(serviceIdentifier) .build() } // add strato putters, fetchers, scanners below: @Provides @Singleton @Named(GuiceNamedConstants.COSINE_FOLLOW_FETCHER) def cosineFollowFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] = stratoClient.fetcher[Long, Unit, HermitCandidates](CosineFollowPath) @Provides @Singleton @Named(GuiceNamedConstants.COSINE_LIST_FETCHER) def cosineListFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] = stratoClient.fetcher[Long, Unit, HermitCandidates](CosineListPath) @Provides @Singleton @Named(GuiceNamedConstants.CURATED_COMPETITOR_ACCOUNTS_FETCHER) def curatedBlacklistedAccountsFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] = stratoClient.fetcher[String, Unit, Seq[Long]](CuratedFilteredAccountsPath) @Provides @Singleton @Named(GuiceNamedConstants.CURATED_CANDIDATES_FETCHER) def curatedCandidatesFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] = stratoClient.fetcher[String, Unit, Seq[Long]](CuratedCandidatesPath) @Provides @Singleton @Named(GuiceNamedConstants.POP_USERS_IN_PLACE_FETCHER) def popUsersInPlaceFetcher(stratoClient: Client): Fetcher[String, Unit, PopUsersInPlace] = stratoClient.fetcher[String, Unit, PopUsersInPlace](PopUsersInPlacePath) @Provides @Singleton @Named(GuiceNamedConstants.RELATABLE_ACCOUNTS_FETCHER) def relatableAccountsFetcher(stratoClient: Client): Fetcher[String, Unit, RelatableAccounts] = stratoClient.fetcher[String, Unit, RelatableAccounts](RelatableAccountsPath) @Provides @Singleton @Named(GuiceNamedConstants.PROFILE_SIDEBAR_BLACKLIST_SCANNER) def profileSidebarBlacklistScanner( stratoClient: Client ): Scanner[(Long, Slice[Long]), Unit, (Long, Long), Unit] = stratoClient.scanner[(Long, Slice[Long]), Unit, (Long, Long), Unit](ProfileSidebarBlacklistPath) @Provides @Singleton @Named(GuiceNamedConstants.REAL_TIME_INTERACTIONS_FETCHER) def realTimeInteractionsFetcher( stratoClient: Client ): Fetcher[(Long, Long), Unit, Seq[Interaction]] = stratoClient.fetcher[(Long, Long), Unit, Seq[Interaction]](RealTimeInteractionsPath) @Provides @Singleton @Named(GuiceNamedConstants.SIMS_FETCHER) def simsFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] = stratoClient.fetcher[Long, Unit, HermitCandidates](SimsPath) @Provides @Singleton @Named(GuiceNamedConstants.DBV2_SIMS_FETCHER) def dbv2SimsFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] = stratoClient.fetcher[Long, Unit, HermitCandidates](DBV2SimsPath) @Provides @Singleton @Named(GuiceNamedConstants.TRIANGULAR_LOOPS_FETCHER) def triangularLoopsFetcher(stratoClient: Client): Fetcher[Long, Unit, TriangularLoopCandidates] = stratoClient.fetcher[Long, Unit, TriangularLoopCandidates](TriangularLoopsPath) @Provides @Singleton @Named(GuiceNamedConstants.TWO_HOP_RANDOM_WALK_FETCHER) def twoHopRandomWalkFetcher(stratoClient: Client): Fetcher[Long, Unit, CandidateSeq] = stratoClient.fetcher[Long, Unit, CandidateSeq](TwoHopRandomWalkPath) @Provides @Singleton @Named(GuiceNamedConstants.USER_RECOMMENDABILITY_FETCHER) def userRecommendabilityFetcher( stratoClient: Client ): Fetcher[Long, Unit, UserRecommendabilityFeatures] = stratoClient.fetcher[Long, Unit, UserRecommendabilityFeatures](UserRecommendabilityPath) @Provides @Singleton @Named(GuiceNamedConstants.USER_STATE_FETCHER) def userStateFetcher(stratoClient: Client): Fetcher[Long, Unit, CondensedUserState] = stratoClient.fetcher[Long, Unit, CondensedUserState](UserStatePath) @Provides @Singleton @Named(GuiceNamedConstants.UTT_ACCOUNT_RECOMMENDATIONS_FETCHER) def uttAccountRecommendationsFetcher( stratoClient: Client ): Fetcher[UTTInterest, Unit, InterestBasedUserRecommendations] = stratoClient.fetcher[UTTInterest, Unit, InterestBasedUserRecommendations]( UTTAccountRecommendationsPath) @Provides @Singleton @Named(GuiceNamedConstants.UTT_SEED_ACCOUNTS_FETCHER) def uttSeedAccountRecommendationsFetcher( stratoClient: Client ): Fetcher[UTTInterest, Unit, InterestBasedUserRecommendations] = stratoClient.fetcher[UTTInterest, Unit, InterestBasedUserRecommendations]( UttSeedAccountsRecommendationPath) @Provides @Singleton @Named(GuiceNamedConstants.ELECTION_CANDIDATES_FETCHER) def electionCandidatesFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] = stratoClient.fetcher[String, Unit, Seq[Long]](ElectionCandidatesPath) @Provides @Singleton @Named(GuiceNamedConstants.USER_USER_GRAPH_FETCHER) def userUserGraphFetcher( stratoClient: Client ): Fetcher[RecommendUserRequest, Unit, RecommendUserResponse] = stratoClient.fetcher[RecommendUserRequest, Unit, RecommendUserResponse](UserUserGraphPath) @Provides @Singleton @Named(GuiceNamedConstants.POST_NUX_WTF_FEATURES_FETCHER) def wtfPostNuxFeaturesFetcher(stratoClient: Client): Fetcher[Long, Unit, CandidateFeatures] = { val attribution = ManhattanAppId("starbuck", "wtf_starbuck") stratoClient .fetcher[Long, Unit, CandidateFeatures](WTFPostNuxFeaturesPath) .withAttribution(attribution) } @Provides @Singleton @Named(GuiceNamedConstants.EXTENDED_NETWORK) def extendedNetworkFetcher( stratoClient: Client ): Fetcher[ExtendedNetworkUserKey, Unit, ExtendedNetworkUserVal] = { stratoClient .fetcher[ExtendedNetworkUserKey, Unit, ExtendedNetworkUserVal](ExtendedNetworkCandidatesPath) } @Provides @Singleton @Named(GuiceNamedConstants.DISMISS_STORE_SCANNER) def dismissStoreScanner( stratoClient: Client ): Scanner[ (Long, Slice[(Long, Long)]), Unit, (Long, (Long, Long)), WhoToFollowDismissEventDetails ] = stratoClient.scanner[ (Long, Slice[(Long, Long)]), // PKEY: userId, LKEY: (-ts, candidateId) Unit, (Long, (Long, Long)), WhoToFollowDismissEventDetails ](WtfDissmissEventsPath) @Provides @Singleton @Named(GuiceNamedConstants.LABELED_NOTIFICATION_FETCHER) def labeledNotificationFetcher( stratoClient: Client ): Fetcher[Long, Unit, LatestEvents] = { stratoClient .fetcher[Long, Unit, LatestEvents](LabeledNotificationPath) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", "stitch/stitch-core", "strato/src/main/scala/com/twitter/strato/client", "user-signal-service/thrift/src/main/thrift:thrift-scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/UserStateClient.scala ================================================ package com.twitter.follow_recommendations.common.clients.user_state import com.google.inject.name.Named import com.twitter.conversions.DurationOps._ import com.twitter.core_workflows.user_model.thriftscala.CondensedUserState import com.twitter.core_workflows.user_model.thriftscala.UserState import com.twitter.decider.Decider import com.twitter.decider.RandomRecipient import com.twitter.finagle.Memcached.Client import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.util.DefaultTimer import com.twitter.follow_recommendations.common.base.StatsUtil import com.twitter.follow_recommendations.common.clients.cache.MemcacheClient import com.twitter.follow_recommendations.common.clients.cache.ThriftEnumOptionBijection import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.follow_recommendations.configapi.deciders.DeciderKey import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.util.Duration import javax.inject.Inject import javax.inject.Singleton import java.lang.{Long => JLong} @Singleton class UserStateClient @Inject() ( @Named(GuiceNamedConstants.USER_STATE_FETCHER) userStateFetcher: Fetcher[ Long, Unit, CondensedUserState ], client: Client, statsReceiver: StatsReceiver, decider: Decider = Decider.False) { private val stats: StatsReceiver = statsReceiver.scope("user_state_client") // client to memcache cluster val bijection = new ThriftEnumOptionBijection[UserState](UserState.apply) val memcacheClient = MemcacheClient[Option[UserState]]( client = client, dest = "/s/cache/follow_recos_service:twemcaches", valueBijection = bijection, ttl = UserStateClient.CacheTTL, statsReceiver = stats.scope("twemcache") ) def getUserState(userId: Long): Stitch[Option[UserState]] = { val deciderKey: String = DeciderKey.EnableDistributedCaching.toString val enableDistributedCaching: Boolean = decider.isAvailable(deciderKey, Some(RandomRecipient)) val userStateStitch: Stitch[Option[UserState]] = enableDistributedCaching match { // read from memcache case true => memcacheClient.readThrough( // add a key prefix to address cache key collisions key = "UserStateClient" + userId.toString, underlyingCall = () => fetchUserState(userId) ) case false => fetchUserState(userId) } val userStateStitchWithTimeout: Stitch[Option[UserState]] = userStateStitch // set a 150ms timeout limit for user state fetches .within(150.milliseconds)(DefaultTimer) .rescue { case e: Exception => stats.scope("rescued").counter(e.getClass.getSimpleName).incr() Stitch(None) } // profile the latency of stitch call and return the result StatsUtil.profileStitch( userStateStitchWithTimeout, stats.scope("getUserState") ) } def fetchUserState(userId: JLong): Stitch[Option[UserState]] = { userStateFetcher.fetch(userId).map(_.v.flatMap(_.userState)) } } object UserStateClient { val CacheTTL: Duration = Duration.fromHours(6) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "util/util-core/src/main/scala/com/twitter/conversions", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/CandidateAlgorithmTypeConstants.scala ================================================ package com.twitter.follow_recommendations.common.constants import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap import com.twitter.hermit.model.Algorithm._ import com.twitter.follow_recommendations.common.models.AlgorithmType object CandidateAlgorithmTypeConstants { /** * Each algorithm is based on one, or more, of the 4 types of information we have on users, * described in [[AlgorithmType]]. Assignment of algorithms to these categories are based on */ private val AlgorithmIdToType: Map[String, Set[AlgorithmType.Value]] = Map( // Activity Algorithms: AlgorithmToFeedbackTokenMap(NewFollowingSimilarUser).toString -> Set(AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(Sims).toString -> Set(AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(NewFollowingSimilarUserSalsa).toString -> Set( AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(RecentEngagementNonDirectFollow).toString -> Set( AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(RecentEngagementSimilarUser).toString -> Set( AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(RecentEngagementSarusOcCur).toString -> Set(AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(RecentSearchBasedRec).toString -> Set(AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(TwistlyTweetAuthors).toString -> Set(AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(Follow2VecNearestNeighbors).toString -> Set(AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(EmailTweetClick).toString -> Set(AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(RepeatedProfileVisits).toString -> Set(AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(GoodTweetClickEngagements).toString -> Set(AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(TweetShareEngagements).toString -> Set(AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(TweetSharerToShareRecipientEngagements).toString -> Set( AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(TweetAuthorToShareRecipientEngagements).toString -> Set( AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(LinearRegressionFollow2VecNearestNeighbors).toString -> Set( AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(NUXLOHistory).toString -> Set(AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(TrafficAttributionAccounts).toString -> Set(AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(RealGraphOonV2).toString -> Set(AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(MagicRecsRecentEngagements).toString -> Set(AlgorithmType.Activity), AlgorithmToFeedbackTokenMap(NotificationEngagement).toString -> Set(AlgorithmType.Activity), // Social Algorithms: AlgorithmToFeedbackTokenMap(TwoHopRandomWalk).toString -> Set(AlgorithmType.Social), AlgorithmToFeedbackTokenMap(RealTimeMutualFollow).toString -> Set(AlgorithmType.Social), AlgorithmToFeedbackTokenMap(ForwardPhoneBook).toString -> Set(AlgorithmType.Social), AlgorithmToFeedbackTokenMap(ForwardEmailBook).toString -> Set(AlgorithmType.Social), AlgorithmToFeedbackTokenMap(NewFollowingNewFollowingExpansion).toString -> Set( AlgorithmType.Social), AlgorithmToFeedbackTokenMap(NewFollowingSarusCoOccurSocialProof).toString -> Set( AlgorithmType.Social), AlgorithmToFeedbackTokenMap(ReverseEmailBookIbis).toString -> Set(AlgorithmType.Social), AlgorithmToFeedbackTokenMap(ReversePhoneBook).toString -> Set(AlgorithmType.Social), AlgorithmToFeedbackTokenMap(StrongTiePredictionRec).toString -> Set(AlgorithmType.Social), AlgorithmToFeedbackTokenMap(StrongTiePredictionRecWithSocialProof).toString -> Set( AlgorithmType.Social), AlgorithmToFeedbackTokenMap(OnlineStrongTiePredictionRec).toString -> Set(AlgorithmType.Social), AlgorithmToFeedbackTokenMap(OnlineStrongTiePredictionRecNoCaching).toString -> Set( AlgorithmType.Social), AlgorithmToFeedbackTokenMap(TriangularLoop).toString -> Set(AlgorithmType.Social), AlgorithmToFeedbackTokenMap(StrongTiePredictionPmi).toString -> Set(AlgorithmType.Social), AlgorithmToFeedbackTokenMap(OnlineStrongTiePredictionRAB).toString -> Set(AlgorithmType.Social), // Geo Algorithms: AlgorithmToFeedbackTokenMap(PopCountryBackFill).toString -> Set(AlgorithmType.Geo), AlgorithmToFeedbackTokenMap(PopCountry).toString -> Set(AlgorithmType.Geo), AlgorithmToFeedbackTokenMap(PopGeohash).toString -> Set(AlgorithmType.Geo), // AlgorithmToFeedbackTokenMap(PopGeohashRealGraph).toString -> Set(AlgorithmType.Geo), AlgorithmToFeedbackTokenMap(EngagedFollowerRatio).toString -> Set(AlgorithmType.Geo), AlgorithmToFeedbackTokenMap(CrowdSearchAccounts).toString -> Set(AlgorithmType.Geo), AlgorithmToFeedbackTokenMap(OrganicFollowAccounts).toString -> Set(AlgorithmType.Geo), AlgorithmToFeedbackTokenMap(PopGeohashQualityFollow).toString -> Set(AlgorithmType.Geo), AlgorithmToFeedbackTokenMap(PPMILocaleFollow).toString -> Set(AlgorithmType.Geo), // Interest Algorithms: AlgorithmToFeedbackTokenMap(TttInterest).toString -> Set(AlgorithmType.Interest), AlgorithmToFeedbackTokenMap(UttInterestRelatedUsers).toString -> Set(AlgorithmType.Interest), AlgorithmToFeedbackTokenMap(UttSeedAccounts).toString -> Set(AlgorithmType.Interest), AlgorithmToFeedbackTokenMap(UttProducerExpansion).toString -> Set(AlgorithmType.Interest), // Hybrid (more than one type) Algorithms: AlgorithmToFeedbackTokenMap(UttProducerOfflineMbcgV1).toString -> Set( AlgorithmType.Interest, AlgorithmType.Geo), AlgorithmToFeedbackTokenMap(CuratedAccounts).toString -> Set( AlgorithmType.Interest, AlgorithmType.Geo), AlgorithmToFeedbackTokenMap(UserUserGraph).toString -> Set( AlgorithmType.Social, AlgorithmType.Activity), ) def getAlgorithmTypes(algoId: String): Set[String] = { AlgorithmIdToType.get(algoId).map(_.map(_.toString)).getOrElse(Set.empty) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/GuiceNamedConstants.scala ================================================ package com.twitter.follow_recommendations.common.constants object GuiceNamedConstants { final val PRODUCER_SIDE_FEATURE_SWITCHES = "PRODUCER_SIDE_FEATURE_SWITCHES" final val CLIENT_EVENT_LOGGER = "CLIENT_EVENT_LOGGER" final val COSINE_FOLLOW_FETCHER = "cosine_follow_fetcher" final val COSINE_LIST_FETCHER = "cosine_list_fetcher" final val CURATED_CANDIDATES_FETCHER = "curated_candidates_fetcher" final val CURATED_COMPETITOR_ACCOUNTS_FETCHER = "curated_competitor_accounts_fetcher" final val POP_USERS_IN_PLACE_FETCHER = "pop_users_in_place_fetcher" final val PROFILE_SIDEBAR_BLACKLIST_SCANNER = "profile_sidebar_blacklist_scanner" final val REQUEST_LOGGER = "REQUEST_LOGGER" final val FLOW_LOGGER = "FLOW_LOGGER" final val REAL_TIME_INTERACTIONS_FETCHER = "real_time_interactions_fetcher" final val SIMS_FETCHER = "sims_fetcher" final val DBV2_SIMS_FETCHER = "dbv2_sims_fetcher" final val TRIANGULAR_LOOPS_FETCHER = "triangular_loops_fetcher" final val TWO_HOP_RANDOM_WALK_FETCHER = "two_hop_random_walk_fetcher" final val USER_RECOMMENDABILITY_FETCHER = "user_recommendability_fetcher" final val USER_STATE_FETCHER = "user_state_fetcher" final val UTT_ACCOUNT_RECOMMENDATIONS_FETCHER = "utt_account_recomendations_fetcher" final val UTT_SEED_ACCOUNTS_FETCHER = "utt_seed_accounts_fetcher" final val ELECTION_CANDIDATES_FETCHER = "election_candidates_fetcher" final val POST_NUX_WTF_FEATURES_FETCHER = "post_nux_wtf_features_fetcher" final val USER_USER_GRAPH_FETCHER = "user_user_graph_fetcher" final val DISMISS_STORE_SCANNER = "dismiss_store_scanner" final val LABELED_NOTIFICATION_FETCHER = "labeled_notification_scanner" final val STP_EP_SCORER = "stp_ep_scorer" final val STP_DBV2_SCORER = "stp_dbv2_scorer" final val STP_RAB_DBV2_SCORER = "stp_rab_dbv2_scorer" final val EXTENDED_NETWORK = "extended_network_candidates" // scoring client constants final val WTF_PROD_DEEPBIRDV2_CLIENT = "wtf_prod_deepbirdv2_client" // ann clients final val RELATABLE_ACCOUNTS_FETCHER = "relatable_accounts_fetcher" } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/ServiceConstants.scala ================================================ package com.twitter.follow_recommendations.common.constants import com.twitter.conversions.StorageUnitOps._ object ServiceConstants { /** thrift client response size limits * these were estimated using monitoring dashboard * 3MB network usage per second / 25 rps ~ 120KB/req << 1MB * we give some buffer here in case some requests require more data than others */ val StringLengthLimit: Long = 10.megabyte.inBytes val ContainerLengthLimit: Long = 1.megabyte.inBytes } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ ":candidate-algorithm-adapter", ":client-context-adapter", ":post-nux-algorithm-adapter", ":pre-fetched-feature-adapter", ], ) target( name = "common", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/slf4j:slf4j-api", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/onboarding/relevance/util/metadata", "util/util-slf4j-api/src/main/scala", ], ) scala_library( name = "candidate-algorithm-adapter", sources = [ "CandidateAlgorithmAdapter.scala", ], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ ":common", "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", ], ) scala_library( name = "client-context-adapter", sources = [ "ClientContextAdapter.scala", ], compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ ":common", "snowflake/src/main/scala/com/twitter/snowflake/id", ], ) scala_library( name = "post-nux-algorithm-adapter", sources = [ "PostNuxAlgorithmAdapter.scala", ], platform = "java8", tags = ["bazel-compatible"], dependencies = [ ":common", "src/scala/com/twitter/ml/featurestore/catalog/features/customer_journey:post-nux-algorithm-aggregate", ], ) scala_library( name = "pre-fetched-feature-adapter", sources = [ "PreFetchedFeatureAdapter.scala", ], platform = "java8", tags = ["bazel-compatible"], dependencies = [ ":common", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/CandidateAlgorithmAdapter.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.adapters import com.twitter.follow_recommendations.common.models.UserCandidateSourceDetails import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap import com.twitter.hermit.model.Algorithm import com.twitter.hermit.model.Algorithm.Algorithm import com.twitter.hermit.model.Algorithm.UttProducerOfflineMbcgV1 import com.twitter.hermit.model.Algorithm.UttProducerOnlineMbcgV1 import com.twitter.ml.api.DataRecord import com.twitter.ml.api.Feature.SparseBinary import com.twitter.ml.api.Feature.SparseContinuous import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.IRecordOneToOneAdapter import com.twitter.ml.api.util.FDsl._ object CandidateAlgorithmAdapter extends IRecordOneToOneAdapter[Option[UserCandidateSourceDetails]] { val CANDIDATE_ALGORITHMS: SparseBinary = new SparseBinary("candidate.source.algorithm_ids") val CANDIDATE_SOURCE_SCORES: SparseContinuous = new SparseContinuous("candidate.source.scores") val CANDIDATE_SOURCE_RANKS: SparseContinuous = new SparseContinuous("candidate.source.ranks") override val getFeatureContext: FeatureContext = new FeatureContext( CANDIDATE_ALGORITHMS, CANDIDATE_SOURCE_SCORES, CANDIDATE_SOURCE_RANKS ) /** list of candidate source remaps to avoid creating different features for experimental sources. * the LHS should contain the experimental source, and the RHS should contain the prod source. */ def remapCandidateSource(a: Algorithm): Algorithm = a match { case UttProducerOnlineMbcgV1 => UttProducerOfflineMbcgV1 case _ => a } // add the list of algorithm feedback tokens (integers) as a sparse binary feature override def adaptToDataRecord( userCandidateSourceDetailsOpt: Option[UserCandidateSourceDetails] ): DataRecord = { val dr = new DataRecord() userCandidateSourceDetailsOpt.foreach { userCandidateSourceDetails => val scoreMap = for { (source, scoreOpt) <- userCandidateSourceDetails.candidateSourceScores score <- scoreOpt algo <- Algorithm.withNameOpt(source.name) algoId <- AlgorithmToFeedbackTokenMap.get(remapCandidateSource(algo)) } yield algoId.toString -> score val rankMap = for { (source, rank) <- userCandidateSourceDetails.candidateSourceRanks algo <- Algorithm.withNameOpt(source.name) algoId <- AlgorithmToFeedbackTokenMap.get(remapCandidateSource(algo)) } yield algoId.toString -> rank.toDouble val algoIds = scoreMap.keys.toSet ++ rankMap.keys.toSet // hydrate if not empty if (rankMap.nonEmpty) { dr.setFeatureValue(CANDIDATE_SOURCE_RANKS, rankMap) } if (scoreMap.nonEmpty) { dr.setFeatureValue(CANDIDATE_SOURCE_SCORES, scoreMap) } if (algoIds.nonEmpty) { dr.setFeatureValue(CANDIDATE_ALGORITHMS, algoIds) } } dr } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/ClientContextAdapter.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.adapters import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.ml.api.Feature.Binary import com.twitter.ml.api.Feature.Continuous import com.twitter.ml.api.Feature.Discrete import com.twitter.ml.api.Feature.Text import com.twitter.ml.api.util.FDsl._ import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.IRecordOneToOneAdapter import com.twitter.onboarding.relevance.util.metadata.LanguageUtil import com.twitter.product_mixer.core.model.marshalling.request.ClientContext import com.twitter.snowflake.id.SnowflakeId object ClientContextAdapter extends IRecordOneToOneAdapter[(ClientContext, DisplayLocation)] { // we name features with `user.account` for relatively static user-related features val USER_COUNTRY: Text = new Text("user.account.country") val USER_LANGUAGE: Text = new Text("user.account.language") // we name features with `user.context` for more dynamic user-related features val USER_LANGUAGE_PREFIX: Text = new Text("user.context.language_prefix") val USER_CLIENT: Discrete = new Discrete("user.context.client") val USER_AGE: Continuous = new Continuous("user.context.age") val USER_IS_RECENT: Binary = new Binary("user.is.recent") // we name features with `meta` for meta info about the WTF recommendation request val META_DISPLAY_LOCATION: Text = new Text("meta.display_location") val META_POSITION: Discrete = new Discrete("meta.position") // This indicates whether a data point is from a random serving policy val META_IS_RANDOM: Binary = new Binary("prediction.engine.is_random") val RECENT_WIN_IN_DAYS: Int = 30 val GOAL_META_POSITION: Long = 1L val GOAL_META_IS_RANDOM: Boolean = true override val getFeatureContext: FeatureContext = new FeatureContext( USER_COUNTRY, USER_LANGUAGE, USER_AGE, USER_LANGUAGE_PREFIX, USER_CLIENT, USER_IS_RECENT, META_DISPLAY_LOCATION, META_POSITION, META_IS_RANDOM ) /** * we only want to set the relevant fields iff they exist to eliminate redundant information * we do some simple normalization on the language code * we set META_POSITION to 1 always * we set META_IS_RANDOM to true always to simulate a random serving distribution * @param record ClientContext and DisplayLocation from the request */ override def adaptToDataRecord(target: (ClientContext, DisplayLocation)): DataRecord = { val dr = new DataRecord() val cc = target._1 val dl = target._2 cc.countryCode.foreach(countryCode => dr.setFeatureValue(USER_COUNTRY, countryCode)) cc.languageCode.foreach(rawLanguageCode => { val userLanguage = LanguageUtil.simplifyLanguage(rawLanguageCode) val userLanguagePrefix = userLanguage.take(2) dr.setFeatureValue(USER_LANGUAGE, userLanguage) dr.setFeatureValue(USER_LANGUAGE_PREFIX, userLanguagePrefix) }) cc.appId.foreach(appId => dr.setFeatureValue(USER_CLIENT, appId)) cc.userId.foreach(id => SnowflakeId.timeFromIdOpt(id).map { signupTime => val userAge = signupTime.untilNow.inMillis.toDouble dr.setFeatureValue(USER_AGE, userAge) dr.setFeatureValue(USER_IS_RECENT, signupTime.untilNow.inDays <= RECENT_WIN_IN_DAYS) signupTime.untilNow.inDays }) dr.setFeatureValue(META_DISPLAY_LOCATION, dl.toFsName) dr.setFeatureValue(META_POSITION, GOAL_META_POSITION) dr.setFeatureValue(META_IS_RANDOM, GOAL_META_IS_RANDOM) dr } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PostNuxAlgorithmAdapter.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.adapters import com.twitter.ml.api.DataRecord import com.twitter.ml.api.Feature import com.twitter.ml.api.Feature.Continuous import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.IRecordOneToOneAdapter import com.twitter.ml.api.util.FDsl._ import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmFeatures import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmIdAggregateFeatureGroup import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmTypeAggregateFeatureGroup import scala.collection.JavaConverters._ object PostNuxAlgorithmIdAdapter extends PostNuxAlgorithmAdapter { override val PostNuxAlgorithmFeatureGroup: PostNuxAlgorithmFeatures = PostNuxAlgorithmIdAggregateFeatureGroup // To keep the length of feature names reasonable, we remove the prefix added by FeatureStore. override val FeatureStorePrefix: String = "wtf_algorithm_id.customer_journey.post_nux_algorithm_id_aggregate_feature_group." } object PostNuxAlgorithmTypeAdapter extends PostNuxAlgorithmAdapter { override val PostNuxAlgorithmFeatureGroup: PostNuxAlgorithmFeatures = PostNuxAlgorithmTypeAggregateFeatureGroup // To keep the length of feature names reasonable, we remove the prefix added by FeatureStore. override val FeatureStorePrefix: String = "wtf_algorithm_type.customer_journey.post_nux_algorithm_type_aggregate_feature_group." } trait PostNuxAlgorithmAdapter extends IRecordOneToOneAdapter[DataRecord] { val PostNuxAlgorithmFeatureGroup: PostNuxAlgorithmFeatures // The string that is attached to the feature name when it is fetched from feature store. val FeatureStorePrefix: String /** * * This stores transformed aggregate features for PostNux algorithm aggregate features. The * transformation here is log-ratio, where ratio is the raw value divided by # of impressions. */ case class TransformedAlgorithmFeatures( ratioLog: Continuous) { def getFeatures: Seq[Continuous] = Seq(ratioLog) } private def applyFeatureStorePrefix(feature: Continuous) = new Continuous( s"$FeatureStorePrefix${feature.getFeatureName}") // The list of input features WITH the prefix assigned to them by FeatureStore. lazy val allInputFeatures: Seq[Seq[Continuous]] = Seq( PostNuxAlgorithmFeatureGroup.Aggregate7DayFeatures.map(applyFeatureStorePrefix), PostNuxAlgorithmFeatureGroup.Aggregate30DayFeatures.map(applyFeatureStorePrefix) ) // This is a list of the features WITHOUT the prefix assigned to them by FeatureStore. lazy val outputBaseFeatureNames: Seq[Seq[Continuous]] = Seq( PostNuxAlgorithmFeatureGroup.Aggregate7DayFeatures, PostNuxAlgorithmFeatureGroup.Aggregate30DayFeatures ) // We use backend impression to calculate ratio values. lazy val ratioDenominators: Seq[Continuous] = Seq( applyFeatureStorePrefix(PostNuxAlgorithmFeatureGroup.BackendImpressions7Days), applyFeatureStorePrefix(PostNuxAlgorithmFeatureGroup.BackendImpressions30Days) ) /** * A mapping from an original feature's ID to the corresponding set of transformed features. * This is used to compute the transformed features for each of the original ones. */ private lazy val TransformedFeaturesMap: Map[Continuous, TransformedAlgorithmFeatures] = outputBaseFeatureNames.flatten.map { feature => ( // The input feature would have the FeatureStore prefix attached to it. new Continuous(s"$FeatureStorePrefix${feature.getFeatureName}"), // We don't keep the FeatureStore prefix to keep the length of feature names reasonable. TransformedAlgorithmFeatures( new Continuous(s"${feature.getFeatureName}-ratio-log") )) }.toMap /** * Given a denominator, number of impressions, this function returns another function that adds * transformed features (log1p and ratio) of an input feature to a DataRecord. */ private def addTransformedFeaturesToDataRecordFunc( originalDr: DataRecord, numImpressions: Double, ): (DataRecord, Continuous) => DataRecord = { (record: DataRecord, feature: Continuous) => { Option(originalDr.getFeatureValue(feature)) foreach { featureValue => TransformedFeaturesMap.get(feature).foreach { transformedFeatures => record.setFeatureValue( transformedFeatures.ratioLog, // We don't use log1p here since the values are ratios and adding 1 to the _ratio_ would // lead to logarithm of values between 1 and 2, essentially making all values the same. math.log((featureValue + 1) / numImpressions) ) } } record } } /** * @param record: The input record whose PostNuxAlgorithm aggregates are to be transformed. * @return the input [[DataRecord]] with transformed aggregates added. */ override def adaptToDataRecord(record: DataRecord): DataRecord = { if (record.continuousFeatures == null) { // There are no base features available, and hence no transformations. record } else { /** * The `foldLeft` below goes through pairs of (1) Feature groups, such as those calculated over * 7 days or 30 days, and (2) the number of impressions for each of these groups, which is the * denominator when ratio is calculated. */ ratioDenominators .zip(allInputFeatures).foldLeft( /* initial empty DataRecord */ record)( ( /* DataRecord with transformed features up to here */ transformedRecord, /* A tuple with the denominator (#impressions) and features to be transformed */ numImpressionsAndFeatures ) => { val (numImpressionsFeature, features) = numImpressionsAndFeatures Option(record.getFeatureValue(numImpressionsFeature)) match { case Some(numImpressions) if numImpressions > 0.0 => /** * With the number of impressions fixed, we generate a function that adds log-ratio * for each feature in the current [[DataRecord]]. The `foldLeft` goes through all * such features and applies that function while updating the kept DataRecord. */ features.foldLeft(transformedRecord)( addTransformedFeaturesToDataRecordFunc(record, numImpressions)) case _ => transformedRecord } }) } } def getFeatures: Seq[Feature[_]] = TransformedFeaturesMap.values.flatMap(_.getFeatures).toSeq override def getFeatureContext: FeatureContext = new FeatureContext() .addFeatures(this.getFeatures.asJava) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PreFetchedFeatureAdapter.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.adapters import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.ml.api.Feature.Continuous import com.twitter.ml.api.util.FDsl._ import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.IRecordOneToOneAdapter import com.twitter.util.Time /** * This adapter mimics UserRecentWTFImpressionsAndFollowsAdapter (for user) and * RecentWTFImpressionsFeatureAdapter (for candidate) for extracting recent impression * and follow features. This adapter extracts user, candidate, and pair-wise features. */ object PreFetchedFeatureAdapter extends IRecordOneToOneAdapter[ (HasPreFetchedFeature, CandidateUser) ] { // impression features val USER_NUM_RECENT_IMPRESSIONS: Continuous = new Continuous( "user.prefetch.num_recent_impressions" ) val USER_LAST_IMPRESSION_DURATION: Continuous = new Continuous( "user.prefetch.last_impression_duration" ) val CANDIDATE_NUM_RECENT_IMPRESSIONS: Continuous = new Continuous( "user-candidate.prefetch.num_recent_impressions" ) val CANDIDATE_LAST_IMPRESSION_DURATION: Continuous = new Continuous( "user-candidate.prefetch.last_impression_duration" ) // follow features val USER_NUM_RECENT_FOLLOWERS: Continuous = new Continuous( "user.prefetch.num_recent_followers" ) val USER_NUM_RECENT_FOLLOWED_BY: Continuous = new Continuous( "user.prefetch.num_recent_followed_by" ) val USER_NUM_RECENT_MUTUAL_FOLLOWS: Continuous = new Continuous( "user.prefetch.num_recent_mutual_follows" ) // impression + follow features val USER_NUM_RECENT_FOLLOWED_IMPRESSIONS: Continuous = new Continuous( "user.prefetch.num_recent_followed_impression" ) val USER_LAST_FOLLOWED_IMPRESSION_DURATION: Continuous = new Continuous( "user.prefetch.last_followed_impression_duration" ) override def adaptToDataRecord( record: (HasPreFetchedFeature, CandidateUser) ): DataRecord = { val (target, candidate) = record val dr = new DataRecord() val t = Time.now // set impression features for user, optionally for candidate dr.setFeatureValue(USER_NUM_RECENT_IMPRESSIONS, target.numWtfImpressions.toDouble) dr.setFeatureValue( USER_LAST_IMPRESSION_DURATION, (t - target.latestImpressionTime).inMillis.toDouble) target.getCandidateImpressionCounts(candidate.id).foreach { counts => dr.setFeatureValue(CANDIDATE_NUM_RECENT_IMPRESSIONS, counts.toDouble) } target.getCandidateLatestTime(candidate.id).foreach { latestTime: Time => dr.setFeatureValue(CANDIDATE_LAST_IMPRESSION_DURATION, (t - latestTime).inMillis.toDouble) } // set recent follow features for user dr.setFeatureValue(USER_NUM_RECENT_FOLLOWERS, target.numRecentFollowedUserIds.toDouble) dr.setFeatureValue(USER_NUM_RECENT_FOLLOWED_BY, target.numRecentFollowedByUserIds.toDouble) dr.setFeatureValue(USER_NUM_RECENT_MUTUAL_FOLLOWS, target.numRecentMutualFollows.toDouble) dr.setFeatureValue(USER_NUM_RECENT_FOLLOWED_IMPRESSIONS, target.numFollowedImpressions.toDouble) dr.setFeatureValue( USER_LAST_FOLLOWED_IMPRESSION_DURATION, target.lastFollowedImpressionDurationMs.getOrElse(Long.MaxValue).toDouble) dr } override def getFeatureContext: FeatureContext = new FeatureContext( USER_NUM_RECENT_IMPRESSIONS, USER_LAST_IMPRESSION_DURATION, CANDIDATE_NUM_RECENT_IMPRESSIONS, CANDIDATE_LAST_IMPRESSION_DURATION, USER_NUM_RECENT_FOLLOWERS, USER_NUM_RECENT_FOLLOWED_BY, USER_NUM_RECENT_MUTUAL_FOLLOWS, USER_NUM_RECENT_FOLLOWED_IMPRESSIONS, USER_LAST_FOLLOWED_IMPRESSION_DURATION, ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", "src/java/com/twitter/ml/api:api-base", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSource.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.common import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.common.models.HasSimilarToContext import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams trait FeatureSource { def id: FeatureSourceId def featureContext: FeatureContext def hydrateFeatures( target: HasClientContext with HasPreFetchedFeature with HasParams with HasSimilarToContext with HasDisplayLocation, candidates: Seq[CandidateUser] ): Stitch[Map[CandidateUser, DataRecord]] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSourceId.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.common sealed trait FeatureSourceId object FeatureSourceId { object CandidateAlgorithmSourceId extends FeatureSourceId object ClientContextSourceId extends FeatureSourceId object FeatureStoreSourceId extends FeatureSourceId object FeatureStoreTimelinesAuthorSourceId extends FeatureSourceId object FeatureStoreGizmoduckSourceId extends FeatureSourceId object FeatureStoreUserMetricCountsSourceId extends FeatureSourceId object FeatureStoreNotificationSourceId extends FeatureSourceId object FeatureStorePrecomputedNotificationSourceId extends FeatureSourceId object FeatureStorePostNuxAlgorithmSourceId extends FeatureSourceId @deprecated object StratoFeatureHydrationSourceId extends FeatureSourceId object PreFetchedFeatureSourceId extends FeatureSourceId object UserScoringFeatureSourceId extends FeatureSourceId } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/HasPreFetchedFeature.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.common import com.twitter.follow_recommendations.common.models.HasMutualFollowedUserIds import com.twitter.follow_recommendations.common.models.HasWtfImpressions import com.twitter.follow_recommendations.common.models.WtfImpression import com.twitter.util.Time trait HasPreFetchedFeature extends HasMutualFollowedUserIds with HasWtfImpressions { lazy val followedImpressions: Seq[WtfImpression] = { for { wtfImprList <- wtfImpressions.toSeq wtfImpr <- wtfImprList if recentFollowedUserIds.exists(_.contains(wtfImpr.candidateId)) } yield wtfImpr } lazy val numFollowedImpressions: Int = followedImpressions.size lazy val lastFollowedImpressionDurationMs: Option[Long] = { if (followedImpressions.nonEmpty) { Some((Time.now - followedImpressions.map(_.latestTime).max).inMillis) } else None } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/ml/featurestore/catalog/datasets/core:socialgraph", "src/scala/com/twitter/ml/featurestore/catalog/datasets/core:usersource", "src/scala/com/twitter/ml/featurestore/catalog/datasets/onboarding:mc-user-counting", "src/scala/com/twitter/ml/featurestore/catalog/datasets/onboarding:user-wtf-algorithm-aggregate", "src/scala/com/twitter/ml/featurestore/catalog/datasets/onboarding:wtf-impression", "src/scala/com/twitter/ml/featurestore/catalog/datasets/onboarding:wtf-post-nux", "src/scala/com/twitter/ml/featurestore/catalog/datasets/onboarding:wtf-user-algorithm-aggregate", "src/scala/com/twitter/ml/featurestore/catalog/datasets/timelines:timelines-author-features", "src/scala/com/twitter/ml/featurestore/catalog/entities/core", "src/scala/com/twitter/ml/featurestore/catalog/entities/onboarding", "src/scala/com/twitter/ml/featurestore/catalog/features/core:socialgraph", "src/scala/com/twitter/ml/featurestore/catalog/features/core:user", "src/scala/com/twitter/ml/featurestore/catalog/features/interests_discovery:user-topic-relationships", "src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:non-mr-notif-summmaries", "src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:non-mr-notif-summmary-aggregates", "src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:nonmr-ntab-summaries", "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:mc-user-counting", "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:post-nux-offline", "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:post-nux-offline-edge", "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:ratio", "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:simcluster-user-interested-in-candidate-known-for", "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:user-wtf-algorithm-aggregate", "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:wtf-impression", "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:wtf-user-algorithm-aggregate", "src/scala/com/twitter/ml/featurestore/catalog/features/rux:user-resurrection", "src/scala/com/twitter/ml/featurestore/catalog/features/timelines:aggregate", "src/scala/com/twitter/ml/featurestore/lib", "src/scala/com/twitter/ml/featurestore/lib/dynamic", "src/scala/com/twitter/ml/featurestore/lib/embedding", "src/scala/com/twitter/ml/featurestore/lib/feature", "src/scala/com/twitter/ml/featurestore/lib/online", "src/scala/com/twitter/ml/featurestore/lib/params", "src/scala/com/twitter/onboarding/relevance/adapters/features/featurestore", "strato/config/columns/ml/featureStore:featureStore-strato-client", "strato/config/columns/ml/featureStore/onboarding:onboarding-strato-client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/CandidateAlgorithmSource.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.sources import com.google.inject.Inject import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.feature_hydration.adapters.CandidateAlgorithmAdapter import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.common.models.HasSimilarToContext import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams /** * This source only takes features from the candidate's source, * which is all the information we have about the candidate pre-feature-hydration */ @Provides @Singleton class CandidateAlgorithmSource @Inject() (stats: StatsReceiver) extends FeatureSource { override val id: FeatureSourceId = FeatureSourceId.CandidateAlgorithmSourceId override val featureContext: FeatureContext = CandidateAlgorithmAdapter.getFeatureContext override def hydrateFeatures( t: HasClientContext with HasPreFetchedFeature with HasParams with HasSimilarToContext with HasDisplayLocation, // we don't use the target here candidates: Seq[CandidateUser] ): Stitch[Map[CandidateUser, DataRecord]] = { val featureHydrationStats = stats.scope("candidate_alg_source") val hasSourceDetailsStat = featureHydrationStats.counter("has_source_details") val noSourceDetailsStat = featureHydrationStats.counter("no_source_details") val noSourceRankStat = featureHydrationStats.counter("no_source_rank") val hasSourceRankStat = featureHydrationStats.counter("has_source_rank") val noSourceScoreStat = featureHydrationStats.counter("no_source_score") val hasSourceScoreStat = featureHydrationStats.counter("has_source_score") val candidatesToAlgoMap = for { candidate <- candidates } yield { if (candidate.userCandidateSourceDetails.nonEmpty) { hasSourceDetailsStat.incr() candidate.userCandidateSourceDetails.foreach { details => if (details.candidateSourceRanks.isEmpty) { noSourceRankStat.incr() } else { hasSourceRankStat.incr() } if (details.candidateSourceScores.isEmpty) { noSourceScoreStat.incr() } else { hasSourceScoreStat.incr() } } } else { noSourceDetailsStat.incr() } candidate -> CandidateAlgorithmAdapter.adaptToDataRecord(candidate.userCandidateSourceDetails) } Stitch.value(candidatesToAlgoMap.toMap) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/ClientContextSource.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.sources import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.follow_recommendations.common.feature_hydration.adapters.ClientContextAdapter import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.common.models.HasSimilarToContext import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams /** * This source only takes features from the request (e.g. client context, WTF display location) * No external calls are made. */ @Provides @Singleton class ClientContextSource() extends FeatureSource { override val id: FeatureSourceId = FeatureSourceId.ClientContextSourceId override val featureContext: FeatureContext = ClientContextAdapter.getFeatureContext override def hydrateFeatures( t: HasClientContext with HasPreFetchedFeature with HasParams with HasSimilarToContext with HasDisplayLocation, candidates: Seq[CandidateUser] ): Stitch[Map[CandidateUser, DataRecord]] = { Stitch.value( candidates .map(_ -> ((t.clientContext, t.displayLocation))).toMap.mapValues( ClientContextAdapter.adaptToDataRecord)) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.sources import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.timelines.configapi.Param import com.twitter.util.Duration import javax.inject.Inject import javax.inject.Singleton @Singleton class FeatureHydrationSourcesFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( FeatureStoreSourceParams.EnableAlgorithmAggregateFeatures, FeatureStoreSourceParams.EnableAuthorTopicAggregateFeatures, FeatureStoreSourceParams.EnableCandidateClientFeatures, FeatureStoreSourceParams.EnableCandidatePrecomputedNotificationFeatures, FeatureStoreSourceParams.EnableCandidateUserAuthorRealTimeAggregateFeatures, FeatureStoreSourceParams.EnableCandidateUserFeatures, FeatureStoreSourceParams.EnableCandidateUserResurrectionFeatures, FeatureStoreSourceParams.EnableCandidateUserTimelinesAuthorAggregateFeatures, FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors, FeatureStoreSourceParams.EnableSeparateClientForGizmoduck, FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting, FeatureStoreSourceParams.EnableSeparateClientForNotifications, FeatureStoreSourceParams.EnableSimilarToUserFeatures, FeatureStoreSourceParams.EnableTargetUserFeatures, FeatureStoreSourceParams.EnableTargetUserResurrectionFeatures, FeatureStoreSourceParams.EnableTargetUserWtfImpressionFeatures, FeatureStoreSourceParams.EnableTopicAggregateFeatures, FeatureStoreSourceParams.EnableUserCandidateEdgeFeatures, FeatureStoreSourceParams.EnableUserCandidateWtfImpressionCandidateFeatures, FeatureStoreSourceParams.EnableUserClientFeatures, FeatureStoreSourceParams.EnableUserTopicFeatures, FeatureStoreSourceParams.EnableUserWtfAlgEdgeFeatures, ) override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( FeatureStoreSourceParams.GlobalFetchTimeout ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFeatureSwitchKeys.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.sources object FeatureHydrationSourcesFeatureSwitchKeys { val EnableAlgorithmAggregateFeatures = "feature_store_source_enable_algorithm_aggregate_features" val EnableAuthorTopicAggregateFeatures = "feature_store_source_enable_author_topic_aggregate_features" val EnableCandidateClientFeatures = "feature_store_source_enable_candidate_client_features" val EnableCandidateNotificationFeatures = "feature_store_source_enable_candidate_notification_features" val EnableCandidatePrecomputedNotificationFeatures = "feature_store_source_enable_candidate_precomputed_notification_features" val EnableCandidateUserFeatures = "feature_store_source_enable_candidate_user_features" val EnableCandidateUserAuthorRealTimeAggregateFeatures = "feature_store_source_enable_candidate_user_author_rta_features" val EnableCandidateUserResurrectionFeatures = "feature_store_source_enable_candidate_user_resurrection_features" val EnableCandidateUserTimelinesAuthorAggregateFeatures = "feature_store_source_enable_candidate_user_timelines_author_aggregate_features" val EnableSimilarToUserFeatures = "feature_store_source_enable_similar_to_user_features" val EnableTargetUserFeatures = "feature_store_source_enable_target_user_features" val EnableTargetUserUserAuthorUserStateRealTimeAggregatesFeature = "feature_store_source_enable_target_user_user_author_user_state_rta_features" val EnableTargetUserResurrectionFeatures = "feature_store_source_enable_target_user_resurrection_features" val EnableTargetUserWtfImpressionFeatures = "feature_store_source_enable_target_user_wtf_impression_features" val EnableTopicAggregateFeatures = "feature_store_source_enable_topic_aggregate_features" val EnableUserCandidateEdgeFeatures = "feature_store_source_enable_user_candidate_edge_features" val EnableUserCandidateWtfImpressionCandidateFeatures = "feature_store_source_enable_user_candidate_wtf_impression_features" val EnableUserClientFeatures = "feature_store_source_enable_user_client_features" val EnableUserNotificationFeatures = "feature_store_source_enable_user_notification_features" val EnableUserTopicFeatures = "feature_store_source_enable_user_topic_features" val EnableUserWtfAlgEdgeFeatures = "feature_store_source_enable_user_wtf_alg_edge_features" val FeatureHydrationTimeout = "feature_store_source_hydration_timeout_in_millis" val UseSeparateClientForTimelinesAuthor = "feature_store_source_separate_client_for_timelines_author_data" val UseSeparateClientMetricCenterUserCounting = "feature_store_source_separate_client_for_mc_user_counting_data" val UseSeparateClientForNotifications = "feature_store_source_separate_client_for_notifications" val UseSeparateClientForGizmoduck = "feature_store_source_separate_client_for_gizmoduck" } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreFeatures.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.sources import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.ml.featurestore.catalog.entities.core.{Author => AuthorEntity} import com.twitter.ml.featurestore.catalog.entities.core.{AuthorTopic => AuthorTopicEntity} import com.twitter.ml.featurestore.catalog.entities.core.{CandidateUser => CandidateUserEntity} import com.twitter.ml.featurestore.catalog.entities.core.{Topic => TopicEntity} import com.twitter.ml.featurestore.catalog.entities.core.{User => UserEntity} import com.twitter.ml.featurestore.catalog.entities.core.{UserCandidate => UserCandidateEntity} import com.twitter.ml.featurestore.catalog.entities.onboarding.UserWtfAlgorithmEntity import com.twitter.ml.featurestore.catalog.entities.onboarding.{ WtfAlgorithm => WtfAlgorithmIdEntity } import com.twitter.ml.featurestore.catalog.entities.onboarding.{ WtfAlgorithmType => WtfAlgorithmTypeEntity } import com.twitter.ml.featurestore.catalog.features.core.UserClients.FullPrimaryClientVersion import com.twitter.ml.featurestore.catalog.features.core.UserClients.NumClients import com.twitter.ml.featurestore.catalog.features.core.UserClients.PrimaryClient import com.twitter.ml.featurestore.catalog.features.core.UserClients.PrimaryClientVersion import com.twitter.ml.featurestore.catalog.features.core.UserClients.PrimaryDeviceManufacturer import com.twitter.ml.featurestore.catalog.features.core.UserClients.PrimaryMobileSdkVersion import com.twitter.ml.featurestore.catalog.features.core.UserClients.SecondaryClient import com.twitter.ml.featurestore.catalog.features.core.UserCounts.Favorites import com.twitter.ml.featurestore.catalog.features.core.UserCounts.Followers import com.twitter.ml.featurestore.catalog.features.core.UserCounts.Following import com.twitter.ml.featurestore.catalog.features.core.UserCounts.Tweets import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmIdAggregateFeatureGroup import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmTypeAggregateFeatureGroup import com.twitter.ml.featurestore.catalog.features.customer_journey.{Utils => FeatureGroupUtils} import com.twitter.ml.featurestore.catalog.features.interests_discovery.UserTopicRelationships.FollowedTopics import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumFavorites import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumFavoritesReceived import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumFollowBacks import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumFollows import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumFollowsReceived import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumLoginDays import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumLoginTweetImpressions import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumMuteBacks import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumMuted import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumOriginalTweets import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumQualityFollowReceived import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumQuoteRetweets import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumQuoteRetweetsReceived import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumReplies import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumRepliesReceived import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumRetweets import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumRetweetsReceived import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumSpamBlocked import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumSpamBlockedBacks import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumTweetImpressions import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumTweets import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumUnfollowBacks import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumUnfollows import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumUserActiveMinutes import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumWasMutualFollowed import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumWasMutualUnfollowed import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumWasUnfollowed import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.Country import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.FollowersOverFollowingRatio import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.Language import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.MutualFollowsOverFollowersRatio import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.MutualFollowsOverFollowingRatio import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.NumFollowers import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.NumFollowings import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.NumMutualFollows import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.TweepCred import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.UserState import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.HaveSameCountry import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.HaveSameLanguage import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.HaveSameUserState import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.NumFollowersGap import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.NumFollowingsGap import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.NumMutualFollowsGap import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.TweepCredGap import com.twitter.ml.featurestore.catalog.features.onboarding.Ratio.FollowersFollowings import com.twitter.ml.featurestore.catalog.features.onboarding.Ratio.MutualFollowsFollowing import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.HasIntersection import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionCandidateKnownForScore import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionClusterIds import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionUserFavCandidateKnownForScore import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionUserFavScore import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionUserFollowCandidateKnownForScore import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionUserFollowScore import com.twitter.ml.featurestore.catalog.features.onboarding.UserWtfAlgorithmAggregate import com.twitter.ml.featurestore.catalog.features.onboarding.WhoToFollowImpression.HomeTimelineWtfCandidateCounts import com.twitter.ml.featurestore.catalog.features.onboarding.WhoToFollowImpression.HomeTimelineWtfCandidateImpressionCounts import com.twitter.ml.featurestore.catalog.features.onboarding.WhoToFollowImpression.HomeTimelineWtfCandidateImpressionLatestTimestamp import com.twitter.ml.featurestore.catalog.features.onboarding.WhoToFollowImpression.HomeTimelineWtfLatestTimestamp import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowRate import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.Follows import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsTweetFavRate import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsTweetReplies import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsTweetReplyRate import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsTweetRetweetRate import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsTweetRetweets import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsWithTweetFavs import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsWithTweetImpressions import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.HasAnyEngagements import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.HasForwardEngagements import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.HasReverseEngagements import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.Impressions import com.twitter.ml.featurestore.catalog.features.rux.UserResurrection.DaysSinceRecentResurrection import com.twitter.ml.featurestore.catalog.features.timelines.AuthorTopicAggregates import com.twitter.ml.featurestore.catalog.features.timelines.EngagementsReceivedByAuthorRealTimeAggregates import com.twitter.ml.featurestore.catalog.features.timelines.NegativeEngagementsReceivedByAuthorRealTimeAggregates import com.twitter.ml.featurestore.catalog.features.timelines.OriginalAuthorAggregates import com.twitter.ml.featurestore.catalog.features.timelines.TopicEngagementRealTimeAggregates import com.twitter.ml.featurestore.catalog.features.timelines.TopicEngagementUserStateRealTimeAggregates import com.twitter.ml.featurestore.catalog.features.timelines.TopicNegativeEngagementUserStateRealTimeAggregates import com.twitter.ml.featurestore.catalog.features.timelines.UserEngagementAuthorUserStateRealTimeAggregates import com.twitter.ml.featurestore.catalog.features.timelines.UserNegativeEngagementAuthorUserStateRealTimeAggregates import com.twitter.ml.featurestore.lib.EntityId import com.twitter.ml.featurestore.lib.UserId import com.twitter.ml.featurestore.lib.feature.BoundFeature import com.twitter.ml.featurestore.lib.feature.Feature object FeatureStoreFeatures { import FeatureStoreRawFeatures._ ///////////////////////////// Target user features //////////////////////// val targetUserFeatures: Set[BoundFeature[_ <: EntityId, _]] = (userKeyedFeatures ++ userAlgorithmAggregateFeatures).map(_.bind(UserEntity)) val targetUserResurrectionFeatures: Set[BoundFeature[_ <: EntityId, _]] = userResurrectionFeatures.map(_.bind(UserEntity)) val targetUserWtfImpressionFeatures: Set[BoundFeature[_ <: EntityId, _]] = wtfImpressionUserFeatures.map(_.bind(UserEntity)) val targetUserUserAuthorUserStateRealTimeAggregatesFeature: Set[BoundFeature[_ <: EntityId, _]] = userAuthorUserStateRealTimeAggregatesFeature.map(_.bind(UserEntity)) val targetUserStatusFeatures: Set[BoundFeature[_ <: EntityId, _]] = userStatusFeatures.map(_.bind(UserEntity).logarithm1p) val targetUserMetricCountFeatures: Set[BoundFeature[_ <: EntityId, _]] = mcFeatures.map(_.bind(UserEntity).logarithm1p) val targetUserClientFeatures: Set[BoundFeature[_ <: EntityId, _]] = clientFeatures.map(_.bind(UserEntity)) ///////////////////////////// Candidate user features //////////////////////// val candidateUserFeatures: Set[BoundFeature[_ <: EntityId, _]] = userKeyedFeatures.map(_.bind(CandidateUserEntity)) val candidateUserAuthorRealTimeAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = authorAggregateFeatures.map(_.bind(CandidateUserEntity)) val candidateUserResurrectionFeatures: Set[BoundFeature[_ <: EntityId, _]] = userResurrectionFeatures.map(_.bind(CandidateUserEntity)) val candidateUserStatusFeatures: Set[BoundFeature[_ <: EntityId, _]] = userStatusFeatures.map(_.bind(CandidateUserEntity).logarithm1p) val candidateUserTimelinesAuthorAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = Set(timelinesAuthorAggregateFeatures.bind(CandidateUserEntity)) val candidateUserMetricCountFeatures: Set[BoundFeature[_ <: EntityId, _]] = mcFeatures.map(_.bind(CandidateUserEntity).logarithm1p) val candidateUserClientFeatures: Set[BoundFeature[_ <: EntityId, _]] = clientFeatures.map(_.bind(CandidateUserEntity)) val similarToUserFeatures: Set[BoundFeature[_ <: EntityId, _]] = (userKeyedFeatures ++ authorAggregateFeatures).map(_.bind(AuthorEntity)) val similarToUserStatusFeatures: Set[BoundFeature[_ <: EntityId, _]] = userStatusFeatures.map(_.bind(AuthorEntity).logarithm1p) val similarToUserTimelinesAuthorAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = Set(timelinesAuthorAggregateFeatures.bind(AuthorEntity)) val similarToUserMetricCountFeatures: Set[BoundFeature[_ <: EntityId, _]] = mcFeatures.map(_.bind(AuthorEntity).logarithm1p) val userCandidateEdgeFeatures: Set[BoundFeature[_ <: EntityId, _]] = (simclusterUVIntersectionFeatures ++ userCandidatePostNuxEdgeFeatures).map( _.bind(UserCandidateEntity)) val userCandidateWtfImpressionCandidateFeatures: Set[BoundFeature[_ <: EntityId, _]] = wtfImpressionCandidateFeatures.map(_.bind(UserCandidateEntity)) /** * Aggregate features based on candidate source algorithms. */ val postNuxAlgorithmIdAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = Set(PostNuxAlgorithmIdAggregateFeatureGroup.FeaturesAsDataRecord) .map(_.bind(WtfAlgorithmIdEntity)) /** * Aggregate features based on candidate source algorithm types. There are 4 at the moment: * Geo, Social, Activity and Interest. */ val postNuxAlgorithmTypeAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = Set(PostNuxAlgorithmTypeAggregateFeatureGroup.FeaturesAsDataRecord) .map(_.bind(WtfAlgorithmTypeEntity)) // user wtf-Algorithm features val userWtfAlgorithmEdgeFeatures: Set[BoundFeature[_ <: EntityId, _]] = FeatureGroupUtils.getTimelinesAggregationFrameworkCombinedFeatures( UserWtfAlgorithmAggregate, UserWtfAlgorithmEntity, FeatureGroupUtils.getMaxSumAvgAggregate(UserWtfAlgorithmAggregate) ) /** * We have to add the max/sum/avg-aggregated features to the set of all features so that we can * register them using FRS's [[FrsFeatureJsonExporter]]. * * Any additional such aggregated features that are included in [[FeatureStoreSource]] client * should be registered here as well. */ val maxSumAvgAggregatedFeatureContext: FeatureContext = new FeatureContext() .addFeatures( UserWtfAlgorithmAggregate.getSecondaryAggregatedFeatureContext ) // topic features val topicAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = Set( TopicEngagementRealTimeAggregates.FeaturesAsDataRecord, TopicNegativeEngagementUserStateRealTimeAggregates.FeaturesAsDataRecord, TopicEngagementUserStateRealTimeAggregates.FeaturesAsDataRecord ).map(_.bind(TopicEntity)) val userTopicFeatures: Set[BoundFeature[_ <: EntityId, _]] = Set(FollowedTopics.bind(UserEntity)) val authorTopicFeatures: Set[BoundFeature[_ <: EntityId, _]] = Set( AuthorTopicAggregates.FeaturesAsDataRecord.bind(AuthorTopicEntity)) val topicFeatures = topicAggregateFeatures ++ userTopicFeatures ++ authorTopicFeatures } object FeatureStoreRawFeatures { val mcFeatures = Set( NumTweets, NumRetweets, NumOriginalTweets, NumRetweetsReceived, NumFavoritesReceived, NumRepliesReceived, NumQuoteRetweetsReceived, NumFollowsReceived, NumFollowBacks, NumFollows, NumUnfollows, NumUnfollowBacks, NumQualityFollowReceived, NumQuoteRetweets, NumFavorites, NumReplies, NumLoginTweetImpressions, NumTweetImpressions, NumLoginDays, NumUserActiveMinutes, NumMuted, NumSpamBlocked, NumMuteBacks, NumSpamBlockedBacks, NumWasMutualFollowed, NumWasMutualUnfollowed, NumWasUnfollowed ) // based off usersource, and each feature represents the cumulative 'sent' counts val userStatusFeatures = Set( Favorites, Followers, Following, Tweets ) // ratio features created from combining other features val userRatioFeatures = Set(MutualFollowsFollowing, FollowersFollowings) // features related to user login history val userResurrectionFeatures: Set[Feature[UserId, Int]] = Set( DaysSinceRecentResurrection ) // real-time aggregate features borrowed from timelines val authorAggregateFeatures = Set( EngagementsReceivedByAuthorRealTimeAggregates.FeaturesAsDataRecord, NegativeEngagementsReceivedByAuthorRealTimeAggregates.FeaturesAsDataRecord, ) val timelinesAuthorAggregateFeatures = OriginalAuthorAggregates.FeaturesAsDataRecord val userAuthorUserStateRealTimeAggregatesFeature: Set[Feature[UserId, DataRecord]] = Set( UserEngagementAuthorUserStateRealTimeAggregates.FeaturesAsDataRecord, UserNegativeEngagementAuthorUserStateRealTimeAggregates.FeaturesAsDataRecord ) // post nux per-user offline features val userOfflineFeatures = Set( NumFollowings, NumFollowers, NumMutualFollows, TweepCred, UserState, Language, Country, MutualFollowsOverFollowingRatio, MutualFollowsOverFollowersRatio, FollowersOverFollowingRatio, ) // matched post nux offline features between user and candidate val userCandidatePostNuxEdgeFeatures = Set( HaveSameUserState, HaveSameLanguage, HaveSameCountry, NumFollowingsGap, NumFollowersGap, NumMutualFollowsGap, TweepCredGap, ) // user algorithm aggregate features val userAlgorithmAggregateFeatures = Set( Impressions, Follows, FollowRate, FollowsWithTweetImpressions, FollowsWithTweetFavs, FollowsTweetFavRate, FollowsTweetReplies, FollowsTweetReplyRate, FollowsTweetRetweets, FollowsTweetRetweetRate, HasForwardEngagements, HasReverseEngagements, HasAnyEngagements, ) val userKeyedFeatures = userRatioFeatures ++ userOfflineFeatures val wtfImpressionUserFeatures = Set(HomeTimelineWtfCandidateCounts, HomeTimelineWtfLatestTimestamp) val wtfImpressionCandidateFeatures = Set(HomeTimelineWtfCandidateImpressionCounts, HomeTimelineWtfCandidateImpressionLatestTimestamp) val simclusterUVIntersectionFeatures = Set( IntersectionClusterIds, HasIntersection, IntersectionUserFollowScore, IntersectionUserFavScore, IntersectionCandidateKnownForScore, IntersectionUserFollowCandidateKnownForScore, IntersectionUserFavCandidateKnownForScore ) // Client features val clientFeatures = Set( NumClients, PrimaryClient, PrimaryClientVersion, FullPrimaryClientVersion, PrimaryDeviceManufacturer, PrimaryMobileSdkVersion, SecondaryClient ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreGizmoduckSource.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.sources import com.github.benmanes.caffeine.cache.Caffeine import com.google.inject.Inject import com.twitter.finagle.TimeoutException import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.adaptAdditionalFeaturesToDataRecord import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.randomizedTTL import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasSimilarToContext import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.IRecordOneToOneAdapter import com.twitter.ml.featurestore.catalog.datasets.core.UsersourceEntityDataset import com.twitter.ml.featurestore.catalog.entities.core.{Author => AuthorEntity} import com.twitter.ml.featurestore.catalog.entities.core.{AuthorTopic => AuthorTopicEntity} import com.twitter.ml.featurestore.catalog.entities.core.{CandidateUser => CandidateUserEntity} import com.twitter.ml.featurestore.catalog.entities.core.{User => UserEntity} import com.twitter.ml.featurestore.lib.EdgeEntityId import com.twitter.ml.featurestore.lib.EntityId import com.twitter.ml.featurestore.lib.TopicId import com.twitter.ml.featurestore.lib.UserId import com.twitter.ml.featurestore.lib.data.PredictionRecord import com.twitter.ml.featurestore.lib.data.PredictionRecordAdapter import com.twitter.ml.featurestore.lib.dataset.DatasetId import com.twitter.ml.featurestore.lib.dataset.online.Hydrator.HydrationResponse import com.twitter.ml.featurestore.lib.dataset.online.OnlineAccessDataset import com.twitter.ml.featurestore.lib.dynamic.ClientConfig import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient import com.twitter.ml.featurestore.lib.dynamic.DynamicHydrationConfig import com.twitter.ml.featurestore.lib.dynamic.FeatureStoreParamsConfig import com.twitter.ml.featurestore.lib.dynamic.GatedFeatures import com.twitter.ml.featurestore.lib.feature.BoundFeature import com.twitter.ml.featurestore.lib.feature.BoundFeatureSet import com.twitter.ml.featurestore.lib.online.DatasetValuesCache import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest import com.twitter.ml.featurestore.lib.online.OnlineFeatureGenerationStats import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import java.util.concurrent.TimeUnit import com.twitter.conversions.DurationOps._ import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext class FeatureStoreGizmoduckSource @Inject() ( serviceIdentifier: ServiceIdentifier, stats: StatsReceiver) extends FeatureSource { import FeatureStoreGizmoduckSource._ val backupSourceStats = stats.scope("feature_store_hydration_gizmoduck") val adapterStats = backupSourceStats.scope("adapters") override def id: FeatureSourceId = FeatureSourceId.FeatureStoreGizmoduckSourceId override def featureContext: FeatureContext = getFeatureContext val clientConfig: ClientConfig[HasParams] = ClientConfig( dynamicHydrationConfig = dynamicHydrationConfig, featureStoreParamsConfig = FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), /** * The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` * used below takes effect. */ timeoutProvider = Function.const(800.millis), serviceIdentifier = serviceIdentifier ) private val datasetsToCache = Set( UsersourceEntityDataset ).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] private val datasetValuesCache: DatasetValuesCache = DatasetValuesCache( Caffeine .newBuilder() .expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) .maximumSize(DefaultCacheMaxKeys) .build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] .asMap, datasetsToCache, DatasetCacheScope ) private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( clientConfig, backupSourceStats, Set(datasetValuesCache) ) private val adapter: IRecordOneToOneAdapter[PredictionRecord] = PredictionRecordAdapter.oneToOne( BoundFeatureSet(allFeatures), OnlineFeatureGenerationStats(backupSourceStats) ) override def hydrateFeatures( target: HasClientContext with HasPreFetchedFeature with HasParams with HasSimilarToContext with HasDisplayLocation, candidates: Seq[CandidateUser] ): Stitch[Map[CandidateUser, DataRecord]] = { target.getOptionalUserId .map { targetUserId => val featureRequests = candidates.map { candidate => val userEntityId = UserEntity.withId(UserId(targetUserId)) val candidateEntityId = CandidateUserEntity.withId(UserId(candidate.id)) val similarToUserId = target.similarToUserIds.map(id => AuthorEntity.withId(UserId(id))) val topicProof = candidate.reason.flatMap(_.accountProof.flatMap(_.topicProof)) val authorTopicEntity = if (topicProof.isDefined) { backupSourceStats.counter("candidates_with_topic_proof").incr() Set( AuthorTopicEntity.withId( EdgeEntityId(UserId(candidate.id), TopicId(topicProof.get.topicId)))) } else Nil val entities = Seq(userEntityId, candidateEntityId) ++ similarToUserId ++ authorTopicEntity FeatureStoreRequest(entities) } val predictionRecordsFut = dynamicFeatureStoreClient(featureRequests, target) val candidateFeatureMap = predictionRecordsFut.map { predictionRecords => // we can zip predictionRecords with candidates as the order is preserved in the client candidates .zip(predictionRecords).map { case (candidate, predictionRecord) => candidate -> adaptAdditionalFeaturesToDataRecord( adapter.adaptToDataRecord(predictionRecord), adapterStats, FeatureStoreSource.featureAdapters) }.toMap } Stitch .callFuture(candidateFeatureMap) .within(target.params(FeatureStoreSourceParams.GlobalFetchTimeout))( com.twitter.finagle.util.DefaultTimer) .rescue { case _: TimeoutException => Stitch.value(Map.empty[CandidateUser, DataRecord]) } }.getOrElse(Stitch.value(Map.empty[CandidateUser, DataRecord])) } } object FeatureStoreGizmoduckSource { private val DatasetCacheScope = "feature_store_local_cache_gizmoduck" private val DefaultCacheMaxKeys = 20000 val allFeatures: Set[BoundFeature[_ <: EntityId, _]] = FeatureStoreFeatures.candidateUserStatusFeatures ++ FeatureStoreFeatures.similarToUserStatusFeatures ++ FeatureStoreFeatures.targetUserStatusFeatures val getFeatureContext: FeatureContext = BoundFeatureSet(allFeatures).toFeatureContext val dynamicHydrationConfig: DynamicHydrationConfig[HasParams] = DynamicHydrationConfig( Set( GatedFeatures( boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.targetUserStatusFeatures), gate = HasParams .paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck) & HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.candidateUserStatusFeatures), gate = HasParams .paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck) & HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.similarToUserStatusFeatures), gate = HasParams .paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck) & HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) ), )) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreParameters.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.sources import com.twitter.conversions.DurationOps._ import com.twitter.ml.featurestore.catalog.datasets.core.UserMobileSdkDataset import com.twitter.ml.featurestore.catalog.datasets.core.UsersourceEntityDataset import com.twitter.ml.featurestore.catalog.datasets.customer_journey.PostNuxAlgorithmIdAggregateDataset import com.twitter.ml.featurestore.catalog.datasets.customer_journey.PostNuxAlgorithmTypeAggregateDataset import com.twitter.ml.featurestore.catalog.datasets.magicrecs.NotificationSummariesEntityDataset import com.twitter.ml.featurestore.catalog.datasets.onboarding.MetricCenterUserCountingFeaturesDataset import com.twitter.ml.featurestore.catalog.datasets.onboarding.UserWtfAlgorithmAggregateFeaturesDataset import com.twitter.ml.featurestore.catalog.datasets.onboarding.WhoToFollowPostNuxFeaturesDataset import com.twitter.ml.featurestore.catalog.datasets.rux.UserRecentReactivationTimeDataset import com.twitter.ml.featurestore.catalog.datasets.timelines.AuthorFeaturesEntityDataset import com.twitter.ml.featurestore.lib.dataset.DatasetParams import com.twitter.ml.featurestore.lib.dataset.online.BatchingPolicy import com.twitter.ml.featurestore.lib.params.FeatureStoreParams import com.twitter.strato.opcontext.Attribution.ManhattanAppId import com.twitter.strato.opcontext.ServeWithin object FeatureStoreParameters { private val FeatureServiceBatchSize = 100 val featureStoreParams = FeatureStoreParams( global = DatasetParams( serveWithin = Some(ServeWithin(duration = 240.millis, roundTripAllowance = None)), attributions = Seq( ManhattanAppId("omega", "wtf_impression_store"), ManhattanAppId("athena", "wtf_athena"), ManhattanAppId("starbuck", "wtf_starbuck"), ManhattanAppId("apollo", "wtf_apollo") ), batchingPolicy = Some(BatchingPolicy.Isolated(FeatureServiceBatchSize)) ), perDataset = Map( MetricCenterUserCountingFeaturesDataset.id -> DatasetParams( stratoSuffix = Some("onboarding"), batchingPolicy = Some(BatchingPolicy.Isolated(200)) ), UsersourceEntityDataset.id -> DatasetParams( stratoSuffix = Some("onboarding") ), WhoToFollowPostNuxFeaturesDataset.id -> DatasetParams( stratoSuffix = Some("onboarding"), batchingPolicy = Some(BatchingPolicy.Isolated(200)) ), AuthorFeaturesEntityDataset.id -> DatasetParams( stratoSuffix = Some("onboarding"), batchingPolicy = Some(BatchingPolicy.Isolated(10)) ), UserRecentReactivationTimeDataset.id -> DatasetParams( stratoSuffix = None // removed due to low hit rate. we should use a negative cache in the future ), UserWtfAlgorithmAggregateFeaturesDataset.id -> DatasetParams( stratoSuffix = None ), NotificationSummariesEntityDataset.id -> DatasetParams( stratoSuffix = Some("onboarding"), serveWithin = Some(ServeWithin(duration = 45.millis, roundTripAllowance = None)), batchingPolicy = Some(BatchingPolicy.Isolated(10)) ), UserMobileSdkDataset.id -> DatasetParams( stratoSuffix = Some("onboarding") ), PostNuxAlgorithmIdAggregateDataset.id -> DatasetParams( stratoSuffix = Some("onboarding") ), PostNuxAlgorithmTypeAggregateDataset.id -> DatasetParams( stratoSuffix = Some("onboarding") ), ), enableFeatureGenerationStats = true ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStorePostNuxAlgorithmSource.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.sources import com.github.benmanes.caffeine.cache.Caffeine import com.google.inject.Inject import com.twitter.conversions.DurationOps._ import com.twitter.finagle.TimeoutException import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.constants.CandidateAlgorithmTypeConstants import com.twitter.follow_recommendations.common.feature_hydration.adapters.CandidateAlgorithmAdapter.remapCandidateSource import com.twitter.follow_recommendations.common.feature_hydration.adapters.PostNuxAlgorithmIdAdapter import com.twitter.follow_recommendations.common.feature_hydration.adapters.PostNuxAlgorithmTypeAdapter import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.adaptAdditionalFeaturesToDataRecord import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.randomizedTTL import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.common.models.HasSimilarToContext import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecordMerger import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.IRecordOneToOneAdapter import com.twitter.ml.featurestore.catalog.datasets.customer_journey.PostNuxAlgorithmIdAggregateDataset import com.twitter.ml.featurestore.catalog.datasets.customer_journey.PostNuxAlgorithmTypeAggregateDataset import com.twitter.ml.featurestore.catalog.entities.onboarding.{WtfAlgorithm => OnboardingWtfAlgoId} import com.twitter.ml.featurestore.catalog.entities.onboarding.{ WtfAlgorithmType => OnboardingWtfAlgoType } import com.twitter.ml.featurestore.catalog.features.customer_journey.CombineAllFeaturesPolicy import com.twitter.ml.featurestore.lib.EntityId import com.twitter.ml.featurestore.lib.WtfAlgorithmId import com.twitter.ml.featurestore.lib.WtfAlgorithmType import com.twitter.ml.featurestore.lib.data.PredictionRecord import com.twitter.ml.featurestore.lib.data.PredictionRecordAdapter import com.twitter.ml.featurestore.lib.dataset.DatasetId import com.twitter.ml.featurestore.lib.dataset.online.Hydrator.HydrationResponse import com.twitter.ml.featurestore.lib.dataset.online.OnlineAccessDataset import com.twitter.ml.featurestore.lib.dynamic.ClientConfig import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient import com.twitter.ml.featurestore.lib.dynamic.DynamicHydrationConfig import com.twitter.ml.featurestore.lib.dynamic.FeatureStoreParamsConfig import com.twitter.ml.featurestore.lib.dynamic.GatedFeatures import com.twitter.ml.featurestore.lib.entity.EntityWithId import com.twitter.ml.featurestore.lib.feature.BoundFeature import com.twitter.ml.featurestore.lib.feature.BoundFeatureSet import com.twitter.ml.featurestore.lib.online.DatasetValuesCache import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest import com.twitter.ml.featurestore.lib.online.OnlineFeatureGenerationStats import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import java.util.concurrent.TimeUnit import scala.collection.JavaConverters._ class FeatureStorePostNuxAlgorithmSource @Inject() ( serviceIdentifier: ServiceIdentifier, stats: StatsReceiver) extends FeatureSource { import FeatureStorePostNuxAlgorithmSource._ val backupSourceStats = stats.scope("feature_store_hydration_post_nux_algorithm") val adapterStats = backupSourceStats.scope("adapters") override def id: FeatureSourceId = FeatureSourceId.FeatureStorePostNuxAlgorithmSourceId override def featureContext: FeatureContext = getFeatureContext private val dataRecordMerger = new DataRecordMerger val clientConfig: ClientConfig[HasParams] = ClientConfig( dynamicHydrationConfig = dynamicHydrationConfig, featureStoreParamsConfig = FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), /** * The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` * used below takes effect. */ timeoutProvider = Function.const(800.millis), serviceIdentifier = serviceIdentifier ) private val datasetsToCache = Set( PostNuxAlgorithmIdAggregateDataset, PostNuxAlgorithmTypeAggregateDataset, ).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] private val datasetValuesCache: DatasetValuesCache = DatasetValuesCache( Caffeine .newBuilder() .expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) .maximumSize(DefaultCacheMaxKeys) .build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] .asMap, datasetsToCache, DatasetCacheScope ) private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( clientConfig, backupSourceStats, Set(datasetValuesCache) ) private val adapterToDataRecord: IRecordOneToOneAdapter[PredictionRecord] = PredictionRecordAdapter.oneToOne( BoundFeatureSet(allFeatures), OnlineFeatureGenerationStats(backupSourceStats) ) // These two calculate the rate for each feature by dividing it by the number of impressions, then // apply a log transformation. private val transformAdapters = Seq(PostNuxAlgorithmIdAdapter, PostNuxAlgorithmTypeAdapter) override def hydrateFeatures( target: HasClientContext with HasPreFetchedFeature with HasParams with HasSimilarToContext with HasDisplayLocation, candidates: Seq[CandidateUser] ): Stitch[Map[CandidateUser, DataRecord]] = { target.getOptionalUserId .map { _: Long => val candidateAlgoIdEntities = candidates.map { candidate => candidate.id -> candidate.getAllAlgorithms .flatMap { algo => AlgorithmToFeedbackTokenMap.get(remapCandidateSource(algo)) }.map(algoId => OnboardingWtfAlgoId.withId(WtfAlgorithmId(algoId))) }.toMap val candidateAlgoTypeEntities = candidateAlgoIdEntities.map { case (candidateId, algoIdEntities) => candidateId -> algoIdEntities .map(_.id.algoId) .flatMap(algoId => CandidateAlgorithmTypeConstants.getAlgorithmTypes(algoId.toString)) .distinct .map(algoType => OnboardingWtfAlgoType.withId(WtfAlgorithmType(algoType))) } val entities = { candidateAlgoIdEntities.values.flatten ++ candidateAlgoTypeEntities.values.flatten }.toSeq.distinct val requests = entities.map(entity => FeatureStoreRequest(Seq(entity))) val predictionRecordsFut = dynamicFeatureStoreClient(requests, target) val candidateFeatureMap = predictionRecordsFut.map { predictionRecords: Seq[PredictionRecord] => val entityFeatureMap: Map[EntityWithId[_], DataRecord] = entities .zip(predictionRecords).map { case (entity, predictionRecord) => entity -> adaptAdditionalFeaturesToDataRecord( adapterToDataRecord.adaptToDataRecord(predictionRecord), adapterStats, transformAdapters) }.toMap // In case we have more than one algorithm ID, or type, for a candidate, we merge the // resulting DataRecords using the two merging policies below. val algoIdMergeFn = CombineAllFeaturesPolicy(PostNuxAlgorithmIdAdapter.getFeatures).getMergeFn val algoTypeMergeFn = CombineAllFeaturesPolicy(PostNuxAlgorithmTypeAdapter.getFeatures).getMergeFn val candidateAlgoIdFeaturesMap = candidateAlgoIdEntities.mapValues { entities => val features = entities.flatMap(e => Option(entityFeatureMap.getOrElse(e, null))) algoIdMergeFn(features) } val candidateAlgoTypeFeaturesMap = candidateAlgoTypeEntities.mapValues { entities => val features = entities.flatMap(e => Option(entityFeatureMap.getOrElse(e, null))) algoTypeMergeFn(features) } candidates.map { candidate => val idDrOpt = candidateAlgoIdFeaturesMap.getOrElse(candidate.id, None) val typeDrOpt = candidateAlgoTypeFeaturesMap.getOrElse(candidate.id, None) val featureDr = (idDrOpt, typeDrOpt) match { case (None, Some(typeDataRecord)) => typeDataRecord case (Some(idDataRecord), None) => idDataRecord case (None, None) => new DataRecord() case (Some(idDataRecord), Some(typeDataRecord)) => dataRecordMerger.merge(idDataRecord, typeDataRecord) idDataRecord } candidate -> featureDr }.toMap } Stitch .callFuture(candidateFeatureMap) .within(target.params(FeatureStoreSourceParams.GlobalFetchTimeout))( com.twitter.finagle.util.DefaultTimer) .rescue { case _: TimeoutException => Stitch.value(Map.empty[CandidateUser, DataRecord]) } }.getOrElse(Stitch.value(Map.empty[CandidateUser, DataRecord])) } } object FeatureStorePostNuxAlgorithmSource { private val DatasetCacheScope = "feature_store_local_cache_post_nux_algorithm" private val DefaultCacheMaxKeys = 1000 // Both of these datasets have <50 keys total. val allFeatures: Set[BoundFeature[_ <: EntityId, _]] = FeatureStoreFeatures.postNuxAlgorithmIdAggregateFeatures ++ FeatureStoreFeatures.postNuxAlgorithmTypeAggregateFeatures val algoIdFinalFeatures = CombineAllFeaturesPolicy( PostNuxAlgorithmIdAdapter.getFeatures).outputFeaturesPostMerge.toSeq val algoTypeFinalFeatures = CombineAllFeaturesPolicy( PostNuxAlgorithmTypeAdapter.getFeatures).outputFeaturesPostMerge.toSeq val getFeatureContext: FeatureContext = new FeatureContext().addFeatures((algoIdFinalFeatures ++ algoTypeFinalFeatures).asJava) val dynamicHydrationConfig: DynamicHydrationConfig[HasParams] = DynamicHydrationConfig( Set( GatedFeatures( boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.postNuxAlgorithmIdAggregateFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableAlgorithmAggregateFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.postNuxAlgorithmTypeAggregateFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableAlgorithmAggregateFeatures) ), )) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSource.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.sources import com.github.benmanes.caffeine.cache.Caffeine import com.google.inject.Inject import com.google.inject.Singleton import com.twitter.conversions.DurationOps._ import com.twitter.finagle.TimeoutException import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.feature_hydration.adapters.CandidateAlgorithmAdapter.remapCandidateSource import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.adaptAdditionalFeaturesToDataRecord import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.randomizedTTL import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.common.models.HasSimilarToContext import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.IRecordOneToOneAdapter import com.twitter.ml.featurestore.catalog.datasets.core.UsersourceEntityDataset import com.twitter.ml.featurestore.catalog.datasets.magicrecs.NotificationSummariesEntityDataset import com.twitter.ml.featurestore.catalog.datasets.onboarding.MetricCenterUserCountingFeaturesDataset import com.twitter.ml.featurestore.catalog.datasets.timelines.AuthorFeaturesEntityDataset import com.twitter.ml.featurestore.catalog.entities.core.{Author => AuthorEntity} import com.twitter.ml.featurestore.catalog.entities.core.{AuthorTopic => AuthorTopicEntity} import com.twitter.ml.featurestore.catalog.entities.core.{CandidateUser => CandidateUserEntity} import com.twitter.ml.featurestore.catalog.entities.core.{Topic => TopicEntity} import com.twitter.ml.featurestore.catalog.entities.core.{User => UserEntity} import com.twitter.ml.featurestore.catalog.entities.core.{UserCandidate => UserCandidateEntity} import com.twitter.ml.featurestore.catalog.entities.onboarding.UserWtfAlgorithmEntity import com.twitter.ml.featurestore.lib.data.PredictionRecord import com.twitter.ml.featurestore.lib.data.PredictionRecordAdapter import com.twitter.ml.featurestore.lib.dataset.online.Hydrator.HydrationResponse import com.twitter.ml.featurestore.lib.dataset.online.OnlineAccessDataset import com.twitter.ml.featurestore.lib.dataset.DatasetId import com.twitter.ml.featurestore.lib.dynamic._ import com.twitter.ml.featurestore.lib.feature._ import com.twitter.ml.featurestore.lib.online.DatasetValuesCache import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest import com.twitter.ml.featurestore.lib.online.OnlineFeatureGenerationStats import com.twitter.ml.featurestore.lib.EdgeEntityId import com.twitter.ml.featurestore.lib.EntityId import com.twitter.ml.featurestore.lib.TopicId import com.twitter.ml.featurestore.lib.UserId import com.twitter.ml.featurestore.lib.WtfAlgorithmId import com.twitter.onboarding.relevance.adapters.features.featurestore.CandidateAuthorTopicAggregatesAdapter import com.twitter.onboarding.relevance.adapters.features.featurestore.CandidateTopicEngagementRealTimeAggregatesAdapter import com.twitter.onboarding.relevance.adapters.features.featurestore.CandidateTopicEngagementUserStateRealTimeAggregatesAdapter import com.twitter.onboarding.relevance.adapters.features.featurestore.CandidateTopicNegativeEngagementUserStateRealTimeAggregatesAdapter import com.twitter.onboarding.relevance.adapters.features.featurestore.FeatureStoreAdapter import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import java.util.concurrent.TimeUnit @Singleton class FeatureStoreSource @Inject() ( serviceIdentifier: ServiceIdentifier, stats: StatsReceiver) extends FeatureSource { import FeatureStoreSource._ override val id: FeatureSourceId = FeatureSourceId.FeatureStoreSourceId override val featureContext: FeatureContext = FeatureStoreSource.getFeatureContext val hydrateFeaturesStats = stats.scope("hydrate_features") val adapterStats = stats.scope("adapters") val featureSet: BoundFeatureSet = BoundFeatureSet(FeatureStoreSource.allFeatures) val clientConfig: ClientConfig[HasParams] = ClientConfig( dynamicHydrationConfig = FeatureStoreSource.dynamicHydrationConfig, featureStoreParamsConfig = FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), /** * The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` * used below takes effect. */ timeoutProvider = Function.const(800.millis), serviceIdentifier = serviceIdentifier ) private val datasetsToCache = Set( MetricCenterUserCountingFeaturesDataset, UsersourceEntityDataset, AuthorFeaturesEntityDataset, NotificationSummariesEntityDataset ).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] private val datasetValuesCache: DatasetValuesCache = DatasetValuesCache( Caffeine .newBuilder() .expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) .maximumSize(DefaultCacheMaxKeys) .build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] .asMap, datasetsToCache, DatasetCacheScope ) private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( clientConfig, stats, Set(datasetValuesCache) ) private val adapter: IRecordOneToOneAdapter[PredictionRecord] = PredictionRecordAdapter.oneToOne( BoundFeatureSet(allFeatures), OnlineFeatureGenerationStats(stats) ) override def hydrateFeatures( target: HasClientContext with HasPreFetchedFeature with HasParams with HasSimilarToContext with HasDisplayLocation, candidates: Seq[CandidateUser] ): Stitch[Map[CandidateUser, DataRecord]] = { target.getOptionalUserId .map { targetUserId => val featureRequests = candidates.map { candidate => val userId = UserId(targetUserId) val userEntityId = UserEntity.withId(userId) val candidateEntityId = CandidateUserEntity.withId(UserId(candidate.id)) val userCandidateEdgeEntityId = UserCandidateEntity.withId(EdgeEntityId(userId, UserId(candidate.id))) val similarToUserId = target.similarToUserIds.map(id => AuthorEntity.withId(UserId(id))) val topicProof = candidate.reason.flatMap(_.accountProof.flatMap(_.topicProof)) val topicEntities = if (topicProof.isDefined) { hydrateFeaturesStats.counter("candidates_with_topic_proof").incr() val topicId = topicProof.get.topicId val topicEntityId = TopicEntity.withId(TopicId(topicId)) val authorTopicEntityId = AuthorTopicEntity.withId(EdgeEntityId(UserId(candidate.id), TopicId(topicId))) Seq(topicEntityId, authorTopicEntityId) } else Nil val candidateAlgorithmsWithScores = candidate.getAllAlgorithms val userWtfAlgEdgeEntities = candidateAlgorithmsWithScores.flatMap(algo => { val algoId = AlgorithmToFeedbackTokenMap.get(remapCandidateSource(algo)) algoId.map(id => UserWtfAlgorithmEntity.withId(EdgeEntityId(userId, WtfAlgorithmId(id)))) }) val entities = Seq( userEntityId, candidateEntityId, userCandidateEdgeEntityId) ++ similarToUserId ++ topicEntities ++ userWtfAlgEdgeEntities FeatureStoreRequest(entities) } val predictionRecordsFut = dynamicFeatureStoreClient(featureRequests, target) val candidateFeatureMap = predictionRecordsFut.map { predictionRecords => // we can zip predictionRecords with candidates as the order is preserved in the client candidates .zip(predictionRecords).map { case (candidate, predictionRecord) => candidate -> adaptAdditionalFeaturesToDataRecord( adapter.adaptToDataRecord(predictionRecord), adapterStats, FeatureStoreSource.featureAdapters) }.toMap } Stitch .callFuture(candidateFeatureMap) .within(target.params(FeatureStoreSourceParams.GlobalFetchTimeout))( com.twitter.finagle.util.DefaultTimer) .rescue { case _: TimeoutException => Stitch.value(Map.empty[CandidateUser, DataRecord]) } }.getOrElse(Stitch.value(Map.empty[CandidateUser, DataRecord])) } } // list of features that we will be fetching, even if we are only scribing but not scoring with them object FeatureStoreSource { private val DatasetCacheScope = "feature_store_local_cache" private val DefaultCacheMaxKeys = 70000 import FeatureStoreFeatures._ ///////////////////// ALL hydrated features ///////////////////// val allFeatures: Set[BoundFeature[_ <: EntityId, _]] = //target user targetUserFeatures ++ targetUserUserAuthorUserStateRealTimeAggregatesFeature ++ targetUserResurrectionFeatures ++ targetUserWtfImpressionFeatures ++ targetUserStatusFeatures ++ targetUserMetricCountFeatures ++ //candidate user candidateUserFeatures ++ candidateUserResurrectionFeatures ++ candidateUserAuthorRealTimeAggregateFeatures ++ candidateUserStatusFeatures ++ candidateUserMetricCountFeatures ++ candidateUserTimelinesAuthorAggregateFeatures ++ candidateUserClientFeatures ++ //similar to user similarToUserFeatures ++ similarToUserStatusFeatures ++ similarToUserMetricCountFeatures ++ similarToUserTimelinesAuthorAggregateFeatures ++ //other userCandidateEdgeFeatures ++ userCandidateWtfImpressionCandidateFeatures ++ topicFeatures ++ userWtfAlgorithmEdgeFeatures ++ targetUserClientFeatures val dynamicHydrationConfig: DynamicHydrationConfig[HasParams] = DynamicHydrationConfig( Set( GatedFeatures( boundFeatureSet = BoundFeatureSet(topicAggregateFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableTopicAggregateFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(authorTopicFeatures), gate = HasParams .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors).unary_! & HasParams.paramGate(FeatureStoreSourceParams.EnableAuthorTopicAggregateFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(userTopicFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableUserTopicFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(targetUserFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(targetUserUserAuthorUserStateRealTimeAggregatesFeature), gate = HasParams.paramGate( FeatureStoreSourceParams.EnableTargetUserUserAuthorUserStateRealTimeAggregatesFeature) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(targetUserResurrectionFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserResurrectionFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(targetUserWtfImpressionFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserWtfImpressionFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(targetUserStatusFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck).unary_! & HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(targetUserMetricCountFeatures), gate = HasParams .paramGate( FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting).unary_! & HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(candidateUserFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(candidateUserAuthorRealTimeAggregateFeatures), gate = HasParams.paramGate( FeatureStoreSourceParams.EnableCandidateUserAuthorRealTimeAggregateFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(candidateUserResurrectionFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserResurrectionFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(candidateUserStatusFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck).unary_! & HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(candidateUserTimelinesAuthorAggregateFeatures), gate = HasParams .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors).unary_! & HasParams.paramGate( FeatureStoreSourceParams.EnableCandidateUserTimelinesAuthorAggregateFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(candidateUserMetricCountFeatures), gate = HasParams .paramGate( FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting).unary_! & HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(userCandidateEdgeFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableUserCandidateEdgeFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(userCandidateWtfImpressionCandidateFeatures), gate = HasParams.paramGate( FeatureStoreSourceParams.EnableUserCandidateWtfImpressionCandidateFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(userWtfAlgorithmEdgeFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableUserWtfAlgEdgeFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(similarToUserFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(similarToUserStatusFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck).unary_! & HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(similarToUserTimelinesAuthorAggregateFeatures), gate = HasParams .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors).unary_! & HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(similarToUserMetricCountFeatures), gate = HasParams .paramGate( FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting).unary_! & HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(candidateUserClientFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateClientFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(targetUserClientFeatures), gate = HasParams.paramGate(FeatureStoreSourceParams.EnableUserClientFeatures) ), ) ) // for calibrating features, e.g. add log transformed topic features val featureAdapters: Seq[FeatureStoreAdapter] = Seq( CandidateTopicEngagementRealTimeAggregatesAdapter, CandidateTopicNegativeEngagementUserStateRealTimeAggregatesAdapter, CandidateTopicEngagementUserStateRealTimeAggregatesAdapter, CandidateAuthorTopicAggregatesAdapter ) val additionalFeatureContext: FeatureContext = FeatureContext.merge( featureAdapters .foldRight(new FeatureContext())((adapter, context) => context .addFeatures(adapter.getFeatureContext)) ) val getFeatureContext: FeatureContext = BoundFeatureSet(allFeatures).toFeatureContext .addFeatures(additionalFeatureContext) // The below are aggregated features that are aggregated for a second time over multiple keys. .addFeatures(maxSumAvgAggregatedFeatureContext) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSourceParams.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.sources import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.util.Duration import com.twitter.conversions.DurationOps._ object FeatureStoreSourceParams { case object EnableTopicAggregateFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTopicAggregateFeatures, default = true ) case object EnableAlgorithmAggregateFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableAlgorithmAggregateFeatures, default = false ) case object EnableAuthorTopicAggregateFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableAuthorTopicAggregateFeatures, default = true ) case object EnableUserTopicFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserTopicFeatures, default = false ) case object EnableTargetUserFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserFeatures, default = true ) case object EnableTargetUserUserAuthorUserStateRealTimeAggregatesFeature extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserUserAuthorUserStateRealTimeAggregatesFeature, default = true ) case object EnableTargetUserResurrectionFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserResurrectionFeatures, default = true ) case object EnableTargetUserWtfImpressionFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserWtfImpressionFeatures, default = true ) case object EnableCandidateUserFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserFeatures, default = true ) case object EnableCandidateUserAuthorRealTimeAggregateFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserAuthorRealTimeAggregateFeatures, default = true ) case object EnableCandidateUserResurrectionFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserResurrectionFeatures, default = true ) case object EnableCandidateUserTimelinesAuthorAggregateFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserTimelinesAuthorAggregateFeatures, default = true ) case object EnableUserCandidateEdgeFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserCandidateEdgeFeatures, default = true ) case object EnableUserCandidateWtfImpressionCandidateFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserCandidateWtfImpressionCandidateFeatures, default = true ) case object EnableUserWtfAlgEdgeFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserWtfAlgEdgeFeatures, default = false ) case object EnableSimilarToUserFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableSimilarToUserFeatures, default = true ) case object EnableCandidatePrecomputedNotificationFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidatePrecomputedNotificationFeatures, default = false ) case object EnableCandidateClientFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateClientFeatures, default = false ) case object EnableUserClientFeatures extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserClientFeatures, default = false ) case object EnableSeparateClientForTimelinesAuthors extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientForTimelinesAuthor, default = false ) case object EnableSeparateClientForMetricCenterUserCounting extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientMetricCenterUserCounting, default = false ) case object EnableSeparateClientForNotifications extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientForNotifications, default = false ) case object EnableSeparateClientForGizmoduck extends FSParam[Boolean]( name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientForGizmoduck, default = false ) case object GlobalFetchTimeout extends FSBoundedParam[Duration]( name = FeatureHydrationSourcesFeatureSwitchKeys.FeatureHydrationTimeout, default = 240.millisecond, min = 100.millisecond, max = 400.millisecond) with HasDurationConversion { override def durationConversion: DurationConversion = DurationConversion.FromMillis } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreTimelinesAuthorSource.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.sources import com.github.benmanes.caffeine.cache.Caffeine import com.google.inject.Inject import com.twitter.finagle.TimeoutException import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.adaptAdditionalFeaturesToDataRecord import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.randomizedTTL import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasSimilarToContext import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.IRecordOneToOneAdapter import com.twitter.ml.featurestore.catalog.datasets.timelines.AuthorFeaturesEntityDataset import com.twitter.ml.featurestore.catalog.entities.core.{Author => AuthorEntity} import com.twitter.ml.featurestore.catalog.entities.core.{AuthorTopic => AuthorTopicEntity} import com.twitter.ml.featurestore.catalog.entities.core.{CandidateUser => CandidateUserEntity} import com.twitter.ml.featurestore.catalog.entities.core.{User => UserEntity} import com.twitter.ml.featurestore.lib.EdgeEntityId import com.twitter.ml.featurestore.lib.EntityId import com.twitter.ml.featurestore.lib.TopicId import com.twitter.ml.featurestore.lib.UserId import com.twitter.ml.featurestore.lib.data.PredictionRecord import com.twitter.ml.featurestore.lib.data.PredictionRecordAdapter import com.twitter.ml.featurestore.lib.dataset.DatasetId import com.twitter.ml.featurestore.lib.dataset.online.Hydrator.HydrationResponse import com.twitter.ml.featurestore.lib.dataset.online.OnlineAccessDataset import com.twitter.ml.featurestore.lib.dynamic.ClientConfig import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient import com.twitter.ml.featurestore.lib.dynamic.DynamicHydrationConfig import com.twitter.ml.featurestore.lib.dynamic.FeatureStoreParamsConfig import com.twitter.ml.featurestore.lib.dynamic.GatedFeatures import com.twitter.ml.featurestore.lib.feature.BoundFeature import com.twitter.ml.featurestore.lib.feature.BoundFeatureSet import com.twitter.ml.featurestore.lib.online.DatasetValuesCache import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest import com.twitter.ml.featurestore.lib.online.OnlineFeatureGenerationStats import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import java.util.concurrent.TimeUnit import com.twitter.conversions.DurationOps._ import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext class FeatureStoreTimelinesAuthorSource @Inject() ( serviceIdentifier: ServiceIdentifier, stats: StatsReceiver) extends FeatureSource { import FeatureStoreTimelinesAuthorSource._ val backupSourceStats = stats.scope("feature_store_hydration_timelines_author") val adapterStats = backupSourceStats.scope("adapters") override def id: FeatureSourceId = FeatureSourceId.FeatureStoreTimelinesAuthorSourceId override def featureContext: FeatureContext = getFeatureContext val clientConfig: ClientConfig[HasParams] = ClientConfig( dynamicHydrationConfig = dynamicHydrationConfig, featureStoreParamsConfig = FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), /** * The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` * used below takes effect. */ timeoutProvider = Function.const(800.millis), serviceIdentifier = serviceIdentifier ) private val datasetsToCache = Set( AuthorFeaturesEntityDataset ).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] private val datasetValuesCache: DatasetValuesCache = DatasetValuesCache( Caffeine .newBuilder() .expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) .maximumSize(DefaultCacheMaxKeys) .build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] .asMap, datasetsToCache, DatasetCacheScope ) private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( clientConfig, backupSourceStats, Set(datasetValuesCache) ) private val adapter: IRecordOneToOneAdapter[PredictionRecord] = PredictionRecordAdapter.oneToOne( BoundFeatureSet(allFeatures), OnlineFeatureGenerationStats(backupSourceStats) ) override def hydrateFeatures( target: HasClientContext with HasPreFetchedFeature with HasParams with HasSimilarToContext with HasDisplayLocation, candidates: Seq[CandidateUser] ): Stitch[Map[CandidateUser, DataRecord]] = { target.getOptionalUserId .map { targetUserId => val featureRequests = candidates.map { candidate => val userEntityId = UserEntity.withId(UserId(targetUserId)) val candidateEntityId = CandidateUserEntity.withId(UserId(candidate.id)) val similarToUserId = target.similarToUserIds.map(id => AuthorEntity.withId(UserId(id))) val topicProof = candidate.reason.flatMap(_.accountProof.flatMap(_.topicProof)) val authorTopicEntity = if (topicProof.isDefined) { backupSourceStats.counter("candidates_with_topic_proof").incr() Set( AuthorTopicEntity.withId( EdgeEntityId(UserId(candidate.id), TopicId(topicProof.get.topicId)))) } else Nil val entities = Seq(userEntityId, candidateEntityId) ++ similarToUserId ++ authorTopicEntity FeatureStoreRequest(entities) } val predictionRecordsFut = dynamicFeatureStoreClient(featureRequests, target) val candidateFeatureMap = predictionRecordsFut.map { predictionRecords => // we can zip predictionRecords with candidates as the order is preserved in the client candidates .zip(predictionRecords).map { case (candidate, predictionRecord) => candidate -> adaptAdditionalFeaturesToDataRecord( adapter.adaptToDataRecord(predictionRecord), adapterStats, FeatureStoreSource.featureAdapters) }.toMap } Stitch .callFuture(candidateFeatureMap) .within(target.params(FeatureStoreSourceParams.GlobalFetchTimeout))( com.twitter.finagle.util.DefaultTimer) .rescue { case _: TimeoutException => Stitch.value(Map.empty[CandidateUser, DataRecord]) } }.getOrElse(Stitch.value(Map.empty[CandidateUser, DataRecord])) } } object FeatureStoreTimelinesAuthorSource { private val DatasetCacheScope = "feature_store_local_cache_timelines_author" private val DefaultCacheMaxKeys = 20000 import FeatureStoreFeatures._ val allFeatures: Set[BoundFeature[_ <: EntityId, _]] = similarToUserTimelinesAuthorAggregateFeatures ++ candidateUserTimelinesAuthorAggregateFeatures ++ authorTopicFeatures val getFeatureContext: FeatureContext = BoundFeatureSet(allFeatures).toFeatureContext val dynamicHydrationConfig: DynamicHydrationConfig[HasParams] = DynamicHydrationConfig( Set( GatedFeatures( boundFeatureSet = BoundFeatureSet(authorTopicFeatures), gate = HasParams .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors) & HasParams.paramGate(FeatureStoreSourceParams.EnableAuthorTopicAggregateFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(similarToUserTimelinesAuthorAggregateFeatures), gate = HasParams .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors) & HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(candidateUserTimelinesAuthorAggregateFeatures), gate = HasParams .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors) & HasParams.paramGate( FeatureStoreSourceParams.EnableCandidateUserTimelinesAuthorAggregateFeatures) ), )) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreUserMetricCountsSource.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.sources import com.github.benmanes.caffeine.cache.Caffeine import com.google.inject.Inject import com.twitter.finagle.TimeoutException import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.adaptAdditionalFeaturesToDataRecord import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.randomizedTTL import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasSimilarToContext import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.IRecordOneToOneAdapter import com.twitter.ml.featurestore.catalog.datasets.onboarding.MetricCenterUserCountingFeaturesDataset import com.twitter.ml.featurestore.catalog.entities.core.{Author => AuthorEntity} import com.twitter.ml.featurestore.catalog.entities.core.{AuthorTopic => AuthorTopicEntity} import com.twitter.ml.featurestore.catalog.entities.core.{CandidateUser => CandidateUserEntity} import com.twitter.ml.featurestore.catalog.entities.core.{User => UserEntity} import com.twitter.ml.featurestore.lib.EdgeEntityId import com.twitter.ml.featurestore.lib.EntityId import com.twitter.ml.featurestore.lib.TopicId import com.twitter.ml.featurestore.lib.UserId import com.twitter.ml.featurestore.lib.data.PredictionRecord import com.twitter.ml.featurestore.lib.data.PredictionRecordAdapter import com.twitter.ml.featurestore.lib.dataset.DatasetId import com.twitter.ml.featurestore.lib.dataset.online.Hydrator.HydrationResponse import com.twitter.ml.featurestore.lib.dataset.online.OnlineAccessDataset import com.twitter.ml.featurestore.lib.dynamic.ClientConfig import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient import com.twitter.ml.featurestore.lib.dynamic.DynamicHydrationConfig import com.twitter.ml.featurestore.lib.dynamic.FeatureStoreParamsConfig import com.twitter.ml.featurestore.lib.dynamic.GatedFeatures import com.twitter.ml.featurestore.lib.feature.BoundFeature import com.twitter.ml.featurestore.lib.feature.BoundFeatureSet import com.twitter.ml.featurestore.lib.online.DatasetValuesCache import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest import com.twitter.ml.featurestore.lib.online.OnlineFeatureGenerationStats import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import java.util.concurrent.TimeUnit import com.twitter.conversions.DurationOps._ import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext class FeatureStoreUserMetricCountsSource @Inject() ( serviceIdentifier: ServiceIdentifier, stats: StatsReceiver) extends FeatureSource { import FeatureStoreUserMetricCountsSource._ val backupSourceStats = stats.scope("feature_store_hydration_mc_counting") val adapterStats = backupSourceStats.scope("adapters") override def id: FeatureSourceId = FeatureSourceId.FeatureStoreUserMetricCountsSourceId override def featureContext: FeatureContext = getFeatureContext val clientConfig: ClientConfig[HasParams] = ClientConfig( dynamicHydrationConfig = dynamicHydrationConfig, featureStoreParamsConfig = FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), /** * The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` * used below takes effect. */ timeoutProvider = Function.const(800.millis), serviceIdentifier = serviceIdentifier ) private val datasetsToCache = Set( MetricCenterUserCountingFeaturesDataset ).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] private val datasetValuesCache: DatasetValuesCache = DatasetValuesCache( Caffeine .newBuilder() .expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) .maximumSize(DefaultCacheMaxKeys) .build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] .asMap, datasetsToCache, DatasetCacheScope ) private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( clientConfig, backupSourceStats, Set(datasetValuesCache) ) private val adapter: IRecordOneToOneAdapter[PredictionRecord] = PredictionRecordAdapter.oneToOne( BoundFeatureSet(allFeatures), OnlineFeatureGenerationStats(backupSourceStats) ) override def hydrateFeatures( target: HasClientContext with HasPreFetchedFeature with HasParams with HasSimilarToContext with HasDisplayLocation, candidates: Seq[CandidateUser] ): Stitch[Map[CandidateUser, DataRecord]] = { target.getOptionalUserId .map { targetUserId => val featureRequests = candidates.map { candidate => val userEntityId = UserEntity.withId(UserId(targetUserId)) val candidateEntityId = CandidateUserEntity.withId(UserId(candidate.id)) val similarToUserId = target.similarToUserIds.map(id => AuthorEntity.withId(UserId(id))) val topicProof = candidate.reason.flatMap(_.accountProof.flatMap(_.topicProof)) val authorTopicEntity = if (topicProof.isDefined) { backupSourceStats.counter("candidates_with_topic_proof").incr() Set( AuthorTopicEntity.withId( EdgeEntityId(UserId(candidate.id), TopicId(topicProof.get.topicId)))) } else Nil val entities = Seq(userEntityId, candidateEntityId) ++ similarToUserId ++ authorTopicEntity FeatureStoreRequest(entities) } val predictionRecordsFut = dynamicFeatureStoreClient(featureRequests, target) val candidateFeatureMap = predictionRecordsFut.map { predictionRecords => // we can zip predictionRecords with candidates as the order is preserved in the client candidates .zip(predictionRecords).map { case (candidate, predictionRecord) => candidate -> adaptAdditionalFeaturesToDataRecord( adapter.adaptToDataRecord(predictionRecord), adapterStats, FeatureStoreSource.featureAdapters) }.toMap } Stitch .callFuture(candidateFeatureMap) .within(target.params(FeatureStoreSourceParams.GlobalFetchTimeout))( com.twitter.finagle.util.DefaultTimer) .rescue { case _: TimeoutException => Stitch.value(Map.empty[CandidateUser, DataRecord]) } }.getOrElse(Stitch.value(Map.empty[CandidateUser, DataRecord])) } } object FeatureStoreUserMetricCountsSource { private val DatasetCacheScope = "feature_store_local_cache_mc_user_counting" private val DefaultCacheMaxKeys = 20000 val allFeatures: Set[BoundFeature[_ <: EntityId, _]] = FeatureStoreFeatures.candidateUserMetricCountFeatures ++ FeatureStoreFeatures.similarToUserMetricCountFeatures ++ FeatureStoreFeatures.targetUserMetricCountFeatures val getFeatureContext: FeatureContext = BoundFeatureSet(allFeatures).toFeatureContext val dynamicHydrationConfig: DynamicHydrationConfig[HasParams] = DynamicHydrationConfig( Set( GatedFeatures( boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.targetUserMetricCountFeatures), gate = HasParams .paramGate(FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting) & HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.candidateUserMetricCountFeatures), gate = HasParams .paramGate(FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting) & HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserFeatures) ), GatedFeatures( boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.similarToUserMetricCountFeatures), gate = HasParams .paramGate(FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting) & HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) ), )) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/HydrationSourcesModule.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.sources import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.escherbird.util.stitchcache.StitchCache import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.stitch.Stitch import com.twitter.storage.client.manhattan.bijections.Bijections.BinaryCompactScalaInjection import com.twitter.storage.client.manhattan.bijections.Bijections.LongInjection import com.twitter.storage.client.manhattan.kv.Guarantee import com.twitter.storage.client.manhattan.kv.ManhattanKVClient import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder import com.twitter.storage.client.manhattan.kv.impl.Component import com.twitter.storage.client.manhattan.kv.impl.Component0 import com.twitter.storage.client.manhattan.kv.impl.KeyDescriptor import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor import com.twitter.strato.generated.client.ml.featureStore.McUserCountingOnUserClientColumn import com.twitter.strato.generated.client.ml.featureStore.onboarding.TimelinesAuthorFeaturesOnUserClientColumn import com.twitter.timelines.author_features.v1.thriftscala.AuthorFeatures import com.twitter.conversions.DurationOps._ import com.twitter.onboarding.relevance.features.thriftscala.MCUserCountingFeatures import java.lang.{Long => JLong} import scala.util.Random object HydrationSourcesModule extends TwitterModule { val readFromManhattan = flag( "feature_hydration_enable_reading_from_manhattan", false, "Whether to read the data from Manhattan or Strato") val manhattanAppId = flag("frs_readonly.appId", "ml_features_athena", "RO App Id used by the RO FRS service") val manhattanDestName = flag( "frs_readonly.destName", "/s/manhattan/athena.native-thrift", "manhattan Dest Name used by the RO FRS service") @Provides @Singleton def providesAthenaManhattanClient( serviceIdentifier: ServiceIdentifier ): ManhattanKVEndpoint = { val client = ManhattanKVClient( manhattanAppId(), manhattanDestName(), ManhattanKVClientMtlsParams(serviceIdentifier) ) ManhattanKVEndpointBuilder(client) .defaultGuarantee(Guarantee.Weak) .build() } val manhattanAuthorDataset = "timelines_author_features" private val defaultCacheMaxKeys = 60000 private val cacheTTL = 12.hours private val earlyExpiration = 0.2 val authorKeyDesc = KeyDescriptor(Component(LongInjection), Component0) val authorDatasetKey = authorKeyDesc.withDataset(manhattanAuthorDataset) val authorValDesc = ValueDescriptor(BinaryCompactScalaInjection(AuthorFeatures)) @Provides @Singleton def timelinesAuthorStitchCache( manhattanReadOnlyEndpoint: ManhattanKVEndpoint, timelinesAuthorFeaturesColumn: TimelinesAuthorFeaturesOnUserClientColumn, stats: StatsReceiver ): StitchCache[JLong, Option[AuthorFeatures]] = { val stitchCacheStats = stats .scope("direct_ds_source_feature_hydration_module").scope("timelines_author") val stStat = stitchCacheStats.counter("readFromStrato-each") val mhtStat = stitchCacheStats.counter("readFromManhattan-each") val timelinesAuthorUnderlyingCall = if (readFromManhattan()) { stitchCacheStats.counter("readFromManhattan").incr() val authorCacheUnderlyingManhattanCall: JLong => Stitch[Option[AuthorFeatures]] = id => { mhtStat.incr() val key = authorDatasetKey.withPkey(id) manhattanReadOnlyEndpoint .get(key = key, valueDesc = authorValDesc).map(_.map(value => clearUnsedFieldsForAuthorFeature(value.contents))) } authorCacheUnderlyingManhattanCall } else { stitchCacheStats.counter("readFromStrato").incr() val authorCacheUnderlyingStratoCall: JLong => Stitch[Option[AuthorFeatures]] = id => { stStat.incr() val timelinesAuthorFeaturesFetcher = timelinesAuthorFeaturesColumn.fetcher timelinesAuthorFeaturesFetcher .fetch(id).map(result => result.v.map(clearUnsedFieldsForAuthorFeature)) } authorCacheUnderlyingStratoCall } StitchCache[JLong, Option[AuthorFeatures]]( underlyingCall = timelinesAuthorUnderlyingCall, maxCacheSize = defaultCacheMaxKeys, ttl = randomizedTTL(cacheTTL.inSeconds).seconds, statsReceiver = stitchCacheStats ) } // Not adding manhattan since it didn't seem useful for Author Data, we can add in another phab // if deemed helpful @Provides @Singleton def metricCenterUserCountingStitchCache( mcUserCountingFeaturesColumn: McUserCountingOnUserClientColumn, stats: StatsReceiver ): StitchCache[JLong, Option[MCUserCountingFeatures]] = { val stitchCacheStats = stats .scope("direct_ds_source_feature_hydration_module").scope("mc_user_counting") val stStat = stitchCacheStats.counter("readFromStrato-each") stitchCacheStats.counter("readFromStrato").incr() val mcUserCountingCacheUnderlyingCall: JLong => Stitch[Option[MCUserCountingFeatures]] = id => { stStat.incr() val mcUserCountingFeaturesFetcher = mcUserCountingFeaturesColumn.fetcher mcUserCountingFeaturesFetcher.fetch(id).map(_.v) } StitchCache[JLong, Option[MCUserCountingFeatures]]( underlyingCall = mcUserCountingCacheUnderlyingCall, maxCacheSize = defaultCacheMaxKeys, ttl = randomizedTTL(cacheTTL.inSeconds).seconds, statsReceiver = stitchCacheStats ) } // clear out fields we don't need to save cache space private def clearUnsedFieldsForAuthorFeature(entry: AuthorFeatures): AuthorFeatures = { entry.unsetUserTopics.unsetUserHealth.unsetAuthorCountryCodeAggregates.unsetOriginalAuthorCountryCodeAggregates } // To avoid a cache stampede. See https://en.wikipedia.org/wiki/Cache_stampede private def randomizedTTL(ttl: Long): Long = { (ttl - ttl * earlyExpiration * Random.nextDouble()).toLong } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/PreFetchedFeatureSource.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.sources import com.google.inject.Inject import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.follow_recommendations.common.feature_hydration.adapters.PreFetchedFeatureAdapter import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.common.models.HasSimilarToContext import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams @Provides @Singleton class PreFetchedFeatureSource @Inject() () extends FeatureSource { override def id: FeatureSourceId = FeatureSourceId.PreFetchedFeatureSourceId override def featureContext: FeatureContext = PreFetchedFeatureAdapter.getFeatureContext override def hydrateFeatures( target: HasClientContext with HasPreFetchedFeature with HasParams with HasSimilarToContext with HasDisplayLocation, candidates: Seq[CandidateUser] ): Stitch[Map[CandidateUser, DataRecord]] = { Stitch.value(candidates.map { candidate => candidate -> PreFetchedFeatureAdapter.adaptToDataRecord((target, candidate)) }.toMap) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/UserScoringFeatureSource.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.sources import com.google.inject.Inject import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.common.models.HasSimilarToContext import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecordMerger import com.twitter.ml.api.FeatureContext import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams /** * This source wraps around the separate sources that we hydrate features from * @param featureStoreSource gets features that require a RPC call to feature store * @param stratoFeatureHydrationSource gets features that require a RPC call to strato columns * @param clientContextSource gets features that are already present in the request context * @param candidateAlgorithmSource gets features that are already present from candidate generation * @param preFetchedFeatureSource gets features that were prehydrated (shared in request lifecycle) */ @Provides @Singleton class UserScoringFeatureSource @Inject() ( featureStoreSource: FeatureStoreSource, featureStoreGizmoduckSource: FeatureStoreGizmoduckSource, featureStorePostNuxAlgorithmSource: FeatureStorePostNuxAlgorithmSource, featureStoreTimelinesAuthorSource: FeatureStoreTimelinesAuthorSource, featureStoreUserMetricCountsSource: FeatureStoreUserMetricCountsSource, clientContextSource: ClientContextSource, candidateAlgorithmSource: CandidateAlgorithmSource, preFetchedFeatureSource: PreFetchedFeatureSource) extends FeatureSource { override val id: FeatureSourceId = FeatureSourceId.UserScoringFeatureSourceId override val featureContext: FeatureContext = FeatureContext.merge( featureStoreSource.featureContext, featureStoreGizmoduckSource.featureContext, featureStorePostNuxAlgorithmSource.featureContext, featureStoreTimelinesAuthorSource.featureContext, featureStoreUserMetricCountsSource.featureContext, clientContextSource.featureContext, candidateAlgorithmSource.featureContext, preFetchedFeatureSource.featureContext, ) val sources = Seq( featureStoreSource, featureStorePostNuxAlgorithmSource, featureStoreTimelinesAuthorSource, featureStoreUserMetricCountsSource, featureStoreGizmoduckSource, clientContextSource, candidateAlgorithmSource, preFetchedFeatureSource ) val dataRecordMerger = new DataRecordMerger def hydrateFeatures( target: HasClientContext with HasPreFetchedFeature with HasParams with HasSimilarToContext with HasDisplayLocation, candidates: Seq[CandidateUser] ): Stitch[Map[CandidateUser, DataRecord]] = { Stitch.collect(sources.map(_.hydrateFeatures(target, candidates))).map { featureMaps => (for { candidate <- candidates } yield { val combinedDataRecord = new DataRecord featureMaps .flatMap(_.get(candidate).toSeq).foreach(dataRecordMerger.merge(combinedDataRecord, _)) candidate -> combinedDataRecord }).toMap } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/Utils.scala ================================================ package com.twitter.follow_recommendations.common.feature_hydration.sources import com.twitter.finagle.stats.StatsReceiver import com.twitter.ml.api.DataRecord import com.twitter.ml.api.IRecordOneToOneAdapter import scala.util.Random /** * Helper functions for FeatureStoreSource operations in FRS are available here. */ object Utils { private val EarlyExpiration = 0.2 private[common] def adaptAdditionalFeaturesToDataRecord( record: DataRecord, adapterStats: StatsReceiver, featureAdapters: Seq[IRecordOneToOneAdapter[DataRecord]] ): DataRecord = { featureAdapters.foldRight(record) { (adapter, record) => adapterStats.counter(adapter.getClass.getSimpleName).incr() adapter.adaptToDataRecord(record) } } // To avoid a cache stampede. See https://en.wikipedia.org/wiki/Cache_stampede private[common] def randomizedTTL(ttl: Long): Long = { (ttl - ttl * EarlyExpiration * Random.nextDouble()).toLong } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/LocationFeature.scala ================================================ package com.twitter.follow_recommendations.common.features import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.pipeline.PipelineQuery case object LocationFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Option[GeohashAndCountryCode]] { override val defaultValue: Option[GeohashAndCountryCode] = None } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/TrackingTokenFeature.scala ================================================ package com.twitter.follow_recommendations.common.features import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.pipeline.PipelineQuery case object TrackingTokenFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Int]] { override val defaultValue: Option[Int] = None } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/UserStateFeature.scala ================================================ package com.twitter.follow_recommendations.common.features import com.twitter.core_workflows.user_model.thriftscala.UserState import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.pipeline.PipelineQuery case object UserStateFeature extends Feature[PipelineQuery, Option[UserState]] {} ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AddressBookMetadata.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier /** * contains information if a candidate is from a candidate source generated using the following signals. */ case class AddressBookMetadata( inForwardPhoneBook: Boolean, inReversePhoneBook: Boolean, inForwardEmailBook: Boolean, inReverseEmailBook: Boolean) object AddressBookMetadata { val ForwardPhoneBookCandidateSource = CandidateSourceIdentifier( Algorithm.ForwardPhoneBook.toString) val ReversePhoneBookCandidateSource = CandidateSourceIdentifier( Algorithm.ReversePhoneBook.toString) val ForwardEmailBookCandidateSource = CandidateSourceIdentifier( Algorithm.ForwardEmailBook.toString) val ReverseEmailBookCandidateSource = CandidateSourceIdentifier( Algorithm.ReverseEmailBookIbis.toString) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AlgorithmType.scala ================================================ package com.twitter.follow_recommendations.common.models /** * Each candidate source algorithm could be based on one, or more, of the 4 general type of * information we have on a user: * 1. Social: the user's connections in Twitter's social graph. * 2. Geo: the user's geographical information. * 3. Interest: information on the user's chosen interests. * 4. Activity: information on the user's past activity. * * Note that an algorithm can fall under more than one of these categories. */ object AlgorithmType extends Enumeration { type AlgorithmType = Value val Social: Value = Value("social") val Geo: Value = Value("geo") val Activity: Value = Value("activity") val Interest: Value = Value("interest") } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", "hermit/hermit-core/src/main/scala/com/twitter/hermit/model", "hermit/hermit-ml/src/main/scala/com/twitter/hermit/ml/models", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", "scrooge/scrooge-serializer/src/main/scala", "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/wtf/scalding/jobs/strong_tie_prediction", "src/thrift/com/twitter/ads/adserver:adserver_rpc-scala", "src/thrift/com/twitter/timelines/author_features/user_health:thrift-scala", "user-signal-service/thrift/src/main/thrift:thrift-scala", "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", ], exports = [ "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/CandidateUser.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.follow_recommendations.logging.{thriftscala => offline} import com.twitter.follow_recommendations.{thriftscala => t} import com.twitter.hermit.constants.AlgorithmFeedbackTokens import com.twitter.ml.api.thriftscala.{DataRecord => TDataRecord} import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions import com.twitter.timelines.configapi.HasParams import com.twitter.timelines.configapi.Params import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier trait FollowableEntity extends UniversalNoun[Long] trait Recommendation extends FollowableEntity with HasReason with HasAdMetadata with HasTrackingToken { val score: Option[Double] def toThrift: t.Recommendation def toOfflineThrift: offline.OfflineRecommendation } case class CandidateUser( override val id: Long, override val score: Option[Double] = None, override val reason: Option[Reason] = None, override val userCandidateSourceDetails: Option[UserCandidateSourceDetails] = None, override val adMetadata: Option[AdMetadata] = None, override val trackingToken: Option[TrackingToken] = None, override val dataRecord: Option[RichDataRecord] = None, override val scores: Option[Scores] = None, override val infoPerRankingStage: Option[scala.collection.Map[String, RankingInfo]] = None, override val params: Params = Params.Invalid, override val engagements: Seq[EngagementType] = Nil, override val recommendationFlowIdentifier: Option[String] = None) extends Recommendation with HasUserCandidateSourceDetails with HasDataRecord with HasScores with HasParams with HasEngagements with HasRecommendationFlowIdentifier with HasInfoPerRankingStage { val rankerIdsStr: Option[Seq[String]] = { val strs = scores.map(_.scores.flatMap(_.rankerId.map(_.toString))) if (strs.exists(_.nonEmpty)) strs else None } val thriftDataRecord: Option[TDataRecord] = for { richDataRecord <- dataRecord dr <- richDataRecord.dataRecord } yield { ScalaToJavaDataRecordConversions.javaDataRecord2ScalaDataRecord(dr) } val toOfflineUserThrift: offline.OfflineUserRecommendation = { val scoringDetails = if (userCandidateSourceDetails.isEmpty && score.isEmpty && thriftDataRecord.isEmpty) { None } else { Some( offline.ScoringDetails( candidateSourceDetails = userCandidateSourceDetails.map(_.toOfflineThrift), score = score, dataRecord = thriftDataRecord, rankerIds = rankerIdsStr, infoPerRankingStage = infoPerRankingStage.map(_.mapValues(_.toOfflineThrift)) ) ) } offline .OfflineUserRecommendation( id, reason.map(_.toOfflineThrift), adMetadata.map(_.adImpression), trackingToken.map(_.toOfflineThrift), scoringDetails = scoringDetails ) } override val toOfflineThrift: offline.OfflineRecommendation = offline.OfflineRecommendation.User(toOfflineUserThrift) val toUserThrift: t.UserRecommendation = { val scoringDetails = if (userCandidateSourceDetails.isEmpty && score.isEmpty && thriftDataRecord.isEmpty && scores.isEmpty) { None } else { Some( t.ScoringDetails( candidateSourceDetails = userCandidateSourceDetails.map(_.toThrift), score = score, dataRecord = thriftDataRecord, rankerIds = rankerIdsStr, debugDataRecord = dataRecord.flatMap(_.debugDataRecord), infoPerRankingStage = infoPerRankingStage.map(_.mapValues(_.toThrift)) ) ) } t.UserRecommendation( userId = id, reason = reason.map(_.toThrift), adImpression = adMetadata.map(_.adImpression), trackingInfo = trackingToken.map(TrackingToken.serialize), scoringDetails = scoringDetails, recommendationFlowIdentifier = recommendationFlowIdentifier ) } override val toThrift: t.Recommendation = t.Recommendation.User(toUserThrift) def setFollowProof(followProofOpt: Option[FollowProof]): CandidateUser = { this.copy( reason = reason .map { reason => reason.copy( accountProof = reason.accountProof .map { accountProof => accountProof.copy(followProof = followProofOpt) }.orElse(Some(AccountProof(followProof = followProofOpt))) ) }.orElse(Some(Reason(Some(AccountProof(followProof = followProofOpt))))) ) } def addScore(score: Score): CandidateUser = { val newScores = scores match { case Some(existingScores) => existingScores.copy(scores = existingScores.scores :+ score) case None => Scores(Seq(score)) } this.copy(scores = Some(newScores)) } } object CandidateUser { val DefaultCandidateScore = 1.0 // for converting candidate in ScoringUserRequest def fromUserRecommendation(candidate: t.UserRecommendation): CandidateUser = { // we only use the primary candidate source for now val userCandidateSourceDetails = for { scoringDetails <- candidate.scoringDetails candidateSourceDetails <- scoringDetails.candidateSourceDetails } yield UserCandidateSourceDetails( primaryCandidateSource = candidateSourceDetails.primarySource .flatMap(AlgorithmFeedbackTokens.TokenToAlgorithmMap.get).map { algo => CandidateSourceIdentifier(algo.toString) }, candidateSourceScores = fromThriftScoreMap(candidateSourceDetails.candidateSourceScores), candidateSourceRanks = fromThriftRankMap(candidateSourceDetails.candidateSourceRanks), addressBookMetadata = None ) CandidateUser( id = candidate.userId, score = candidate.scoringDetails.flatMap(_.score), reason = candidate.reason.map(Reason.fromThrift), userCandidateSourceDetails = userCandidateSourceDetails, trackingToken = candidate.trackingInfo.map(TrackingToken.deserialize), recommendationFlowIdentifier = candidate.recommendationFlowIdentifier, infoPerRankingStage = candidate.scoringDetails.flatMap( _.infoPerRankingStage.map(_.mapValues(RankingInfo.fromThrift))) ) } def fromThriftScoreMap( thriftMapOpt: Option[scala.collection.Map[String, Double]] ): Map[CandidateSourceIdentifier, Option[Double]] = { (for { thriftMap <- thriftMapOpt.toSeq (algoName, score) <- thriftMap.toSeq } yield { CandidateSourceIdentifier(algoName) -> Some(score) }).toMap } def fromThriftRankMap( thriftMapOpt: Option[scala.collection.Map[String, Int]] ): Map[CandidateSourceIdentifier, Int] = { (for { thriftMap <- thriftMapOpt.toSeq (algoName, rank) <- thriftMap.toSeq } yield { CandidateSourceIdentifier(algoName) -> rank }).toMap } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/ClientContextConverter.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.follow_recommendations.logging.{thriftscala => offline} import com.twitter.follow_recommendations.{thriftscala => frs} import com.twitter.product_mixer.core.model.marshalling.request.ClientContext object ClientContextConverter { def toFRSOfflineClientContextThrift( productMixerClientContext: ClientContext ): offline.OfflineClientContext = offline.OfflineClientContext( productMixerClientContext.userId, productMixerClientContext.guestId, productMixerClientContext.appId, productMixerClientContext.countryCode, productMixerClientContext.languageCode, productMixerClientContext.guestIdAds, productMixerClientContext.guestIdMarketing ) def fromThrift(clientContext: frs.ClientContext): ClientContext = ClientContext( userId = clientContext.userId, guestId = clientContext.guestId, appId = clientContext.appId, ipAddress = clientContext.ipAddress, userAgent = clientContext.userAgent, countryCode = clientContext.countryCode, languageCode = clientContext.languageCode, isTwoffice = clientContext.isTwoffice, userRoles = clientContext.userRoles.map(_.toSet), deviceId = clientContext.deviceId, guestIdAds = clientContext.guestIdAds, guestIdMarketing = clientContext.guestIdMarketing, mobileDeviceId = None, mobileDeviceAdId = None, limitAdTracking = None ) def toThrift(clientContext: ClientContext): frs.ClientContext = frs.ClientContext( userId = clientContext.userId, guestId = clientContext.guestIdAds, appId = clientContext.appId, ipAddress = clientContext.ipAddress, userAgent = clientContext.userAgent, countryCode = clientContext.countryCode, languageCode = clientContext.languageCode, isTwoffice = clientContext.isTwoffice, userRoles = clientContext.userRoles, deviceId = clientContext.deviceId, guestIdAds = clientContext.guestIdAds, guestIdMarketing = clientContext.guestIdMarketing ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.adserver.thriftscala.{DisplayLocation => AdDisplayLocation} import com.twitter.follow_recommendations.logging.thriftscala.{ OfflineDisplayLocation => TOfflineDisplayLocation } import com.twitter.follow_recommendations.thriftscala.{DisplayLocation => TDisplayLocation} sealed trait DisplayLocation { def toThrift: TDisplayLocation def toOfflineThrift: TOfflineDisplayLocation def toFsName: String // corresponding display location in adserver if available // make sure to be consistent with the definition here def toAdDisplayLocation: Option[AdDisplayLocation] = None } /** * Make sure you add the new DL to the following files and redeploy our attribution jobs * - follow-recommendations-service/thrift/src/main/thrift/display_location.thrift * - follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift * - follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala */ object DisplayLocation { case object ProfileSidebar extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.ProfileSidebar override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.ProfileSidebar override val toFsName: String = "ProfileSidebar" override val toAdDisplayLocation: Option[AdDisplayLocation] = Some( AdDisplayLocation.ProfileAccountsSidebar ) } case object HomeTimeline extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.HomeTimeline override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.HomeTimeline override val toFsName: String = "HomeTimeline" override val toAdDisplayLocation: Option[AdDisplayLocation] = Some( // it is based on the logic that HTL DL should correspond to Sidebar: AdDisplayLocation.WtfSidebar ) } case object ReactiveFollow extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.ReactiveFollow override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.ReactiveFollow override val toFsName: String = "ReactiveFollow" } case object ExploreTab extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.ExploreTab override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.ExploreTab override val toFsName: String = "ExploreTab" } case object MagicRecs extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.MagicRecs override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.MagicRecs override val toFsName: String = "MagicRecs" } case object AbUploadInjection extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.AbUploadInjection override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.AbUploadInjection override val toFsName: String = "AbUploadInjection" } case object RuxLandingPage extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.RuxLandingPage override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.RuxLandingPage override val toFsName: String = "RuxLandingPage" } case object ProfileBonusFollow extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.ProfileBonusFollow override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.ProfileBonusFollow override val toFsName: String = "ProfileBonusFollow" } case object ElectionExploreWtf extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.ElectionExploreWtf override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.ElectionExploreWtf override val toFsName: String = "ElectionExploreWtf" } case object ClusterFollow extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.ClusterFollow override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.ClusterFollow override val toFsName: String = "ClusterFollow" override val toAdDisplayLocation: Option[AdDisplayLocation] = Some( AdDisplayLocation.ClusterFollow ) } case object HtlBonusFollow extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.HtlBonusFollow override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.HtlBonusFollow override val toFsName: String = "HtlBonusFollow" } case object TopicLandingPageHeader extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.TopicLandingPageHeader override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.TopicLandingPageHeader override val toFsName: String = "TopicLandingPageHeader" } case object NewUserSarusBackfill extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.NewUserSarusBackfill override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.NewUserSarusBackfill override val toFsName: String = "NewUserSarusBackfill" } case object NuxPymk extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.NuxPymk override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.NuxPymk override val toFsName: String = "NuxPymk" } case object NuxInterests extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.NuxInterests override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.NuxInterests override val toFsName: String = "NuxInterests" } case object NuxTopicBonusFollow extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.NuxTopicBonusFollow override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.NuxTopicBonusFollow override val toFsName: String = "NuxTopicBonusFollow" } case object Sidebar extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.Sidebar override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.Sidebar override val toFsName: String = "Sidebar" override val toAdDisplayLocation: Option[AdDisplayLocation] = Some( AdDisplayLocation.WtfSidebar ) } case object CampaignForm extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.CampaignForm override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.CampaignForm override val toFsName: String = "CampaignForm" } case object ProfileTopFollowers extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.ProfileTopFollowers override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.ProfileTopFollowers override val toFsName: String = "ProfileTopFollowers" } case object ProfileTopFollowing extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.ProfileTopFollowing override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.ProfileTopFollowing override val toFsName: String = "ProfileTopFollowing" } case object RuxPymk extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.RuxPymk override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.RuxPymk override val toFsName: String = "RuxPymk" } case object IndiaCovid19CuratedAccountsWtf extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.IndiaCovid19CuratedAccountsWtf override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.IndiaCovid19CuratedAccountsWtf override val toFsName: String = "IndiaCovid19CuratedAccountsWtf" } case object PeoplePlusPlus extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.PeoplePlusPlus override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.PeoplePlusPlus override val toFsName: String = "PeoplePlusPlus" } case object TweetNotificationRecs extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.TweetNotificationRecs override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.TweetNotificationRecs override val toFsName: String = "TweetNotificationRecs" } case object ProfileDeviceFollow extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.ProfileDeviceFollow override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.ProfileDeviceFollow override val toFsName: String = "ProfileDeviceFollow" } case object RecosBackfill extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.RecosBackfill override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.RecosBackfill override val toFsName: String = "RecosBackfill" } case object HtlSpaceHosts extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.HtlSpaceHosts override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.HtlSpaceHosts override val toFsName: String = "HtlSpaceHosts" } case object PostNuxFollowTask extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.PostNuxFollowTask override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.PostNuxFollowTask override val toFsName: String = "PostNuxFollowTask" } case object TopicLandingPage extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.TopicLandingPage override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.TopicLandingPage override val toFsName: String = "TopicLandingPage" } case object UserTypeaheadPrefetch extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.UserTypeaheadPrefetch override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.UserTypeaheadPrefetch override val toFsName: String = "UserTypeaheadPrefetch" } case object HomeTimelineRelatableAccounts extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.HomeTimelineRelatableAccounts override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.HomeTimelineRelatableAccounts override val toFsName: String = "HomeTimelineRelatableAccounts" } case object NuxGeoCategory extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.NuxGeoCategory override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.NuxGeoCategory override val toFsName: String = "NuxGeoCategory" } case object NuxInterestsCategory extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.NuxInterestsCategory override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.NuxInterestsCategory override val toFsName: String = "NuxInterestsCategory" } case object TopArticles extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.TopArticles override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.TopArticles override val toFsName: String = "TopArticles" } case object NuxPymkCategory extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.NuxPymkCategory override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.NuxPymkCategory override val toFsName: String = "NuxPymkCategory" } case object HomeTimelineTweetRecs extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.HomeTimelineTweetRecs override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.HomeTimelineTweetRecs override val toFsName: String = "HomeTimelineTweetRecs" } case object HtlBulkFriendFollows extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.HtlBulkFriendFollows override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.HtlBulkFriendFollows override val toFsName: String = "HtlBulkFriendFollows" } case object NuxAutoFollow extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.NuxAutoFollow override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.NuxAutoFollow override val toFsName: String = "NuxAutoFollow" } case object SearchBonusFollow extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.SearchBonusFollow override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.SearchBonusFollow override val toFsName: String = "SearchBonusFollow" } case object ContentRecommender extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.ContentRecommender override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.ContentRecommender override val toFsName: String = "ContentRecommender" } case object HomeTimelineReverseChron extends DisplayLocation { override val toThrift: TDisplayLocation = TDisplayLocation.HomeTimelineReverseChron override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.HomeTimelineReverseChron override val toFsName: String = "HomeTimelineReverseChron" } def fromThrift(displayLocation: TDisplayLocation): DisplayLocation = displayLocation match { case TDisplayLocation.ProfileSidebar => ProfileSidebar case TDisplayLocation.HomeTimeline => HomeTimeline case TDisplayLocation.MagicRecs => MagicRecs case TDisplayLocation.AbUploadInjection => AbUploadInjection case TDisplayLocation.RuxLandingPage => RuxLandingPage case TDisplayLocation.ProfileBonusFollow => ProfileBonusFollow case TDisplayLocation.ElectionExploreWtf => ElectionExploreWtf case TDisplayLocation.ClusterFollow => ClusterFollow case TDisplayLocation.HtlBonusFollow => HtlBonusFollow case TDisplayLocation.ReactiveFollow => ReactiveFollow case TDisplayLocation.TopicLandingPageHeader => TopicLandingPageHeader case TDisplayLocation.NewUserSarusBackfill => NewUserSarusBackfill case TDisplayLocation.NuxPymk => NuxPymk case TDisplayLocation.NuxInterests => NuxInterests case TDisplayLocation.NuxTopicBonusFollow => NuxTopicBonusFollow case TDisplayLocation.ExploreTab => ExploreTab case TDisplayLocation.Sidebar => Sidebar case TDisplayLocation.CampaignForm => CampaignForm case TDisplayLocation.ProfileTopFollowers => ProfileTopFollowers case TDisplayLocation.ProfileTopFollowing => ProfileTopFollowing case TDisplayLocation.RuxPymk => RuxPymk case TDisplayLocation.IndiaCovid19CuratedAccountsWtf => IndiaCovid19CuratedAccountsWtf case TDisplayLocation.PeoplePlusPlus => PeoplePlusPlus case TDisplayLocation.TweetNotificationRecs => TweetNotificationRecs case TDisplayLocation.ProfileDeviceFollow => ProfileDeviceFollow case TDisplayLocation.RecosBackfill => RecosBackfill case TDisplayLocation.HtlSpaceHosts => HtlSpaceHosts case TDisplayLocation.PostNuxFollowTask => PostNuxFollowTask case TDisplayLocation.TopicLandingPage => TopicLandingPage case TDisplayLocation.UserTypeaheadPrefetch => UserTypeaheadPrefetch case TDisplayLocation.HomeTimelineRelatableAccounts => HomeTimelineRelatableAccounts case TDisplayLocation.NuxGeoCategory => NuxGeoCategory case TDisplayLocation.NuxInterestsCategory => NuxInterestsCategory case TDisplayLocation.TopArticles => TopArticles case TDisplayLocation.NuxPymkCategory => NuxPymkCategory case TDisplayLocation.HomeTimelineTweetRecs => HomeTimelineTweetRecs case TDisplayLocation.HtlBulkFriendFollows => HtlBulkFriendFollows case TDisplayLocation.NuxAutoFollow => NuxAutoFollow case TDisplayLocation.SearchBonusFollow => SearchBonusFollow case TDisplayLocation.ContentRecommender => ContentRecommender case TDisplayLocation.HomeTimelineReverseChron => HomeTimelineReverseChron case TDisplayLocation.EnumUnknownDisplayLocation(i) => throw new UnknownDisplayLocationException( s"Unknown display location thrift enum with value: ${i}") } def fromOfflineThrift(displayLocation: TOfflineDisplayLocation): DisplayLocation = displayLocation match { case TOfflineDisplayLocation.ProfileSidebar => ProfileSidebar case TOfflineDisplayLocation.HomeTimeline => HomeTimeline case TOfflineDisplayLocation.MagicRecs => MagicRecs case TOfflineDisplayLocation.AbUploadInjection => AbUploadInjection case TOfflineDisplayLocation.RuxLandingPage => RuxLandingPage case TOfflineDisplayLocation.ProfileBonusFollow => ProfileBonusFollow case TOfflineDisplayLocation.ElectionExploreWtf => ElectionExploreWtf case TOfflineDisplayLocation.ClusterFollow => ClusterFollow case TOfflineDisplayLocation.HtlBonusFollow => HtlBonusFollow case TOfflineDisplayLocation.TopicLandingPageHeader => TopicLandingPageHeader case TOfflineDisplayLocation.NewUserSarusBackfill => NewUserSarusBackfill case TOfflineDisplayLocation.NuxPymk => NuxPymk case TOfflineDisplayLocation.NuxInterests => NuxInterests case TOfflineDisplayLocation.NuxTopicBonusFollow => NuxTopicBonusFollow case TOfflineDisplayLocation.ExploreTab => ExploreTab case TOfflineDisplayLocation.ReactiveFollow => ReactiveFollow case TOfflineDisplayLocation.Sidebar => Sidebar case TOfflineDisplayLocation.CampaignForm => CampaignForm case TOfflineDisplayLocation.ProfileTopFollowers => ProfileTopFollowers case TOfflineDisplayLocation.ProfileTopFollowing => ProfileTopFollowing case TOfflineDisplayLocation.RuxPymk => RuxPymk case TOfflineDisplayLocation.IndiaCovid19CuratedAccountsWtf => IndiaCovid19CuratedAccountsWtf case TOfflineDisplayLocation.PeoplePlusPlus => PeoplePlusPlus case TOfflineDisplayLocation.TweetNotificationRecs => TweetNotificationRecs case TOfflineDisplayLocation.ProfileDeviceFollow => ProfileDeviceFollow case TOfflineDisplayLocation.RecosBackfill => RecosBackfill case TOfflineDisplayLocation.HtlSpaceHosts => HtlSpaceHosts case TOfflineDisplayLocation.PostNuxFollowTask => PostNuxFollowTask case TOfflineDisplayLocation.TopicLandingPage => TopicLandingPage case TOfflineDisplayLocation.UserTypeaheadPrefetch => UserTypeaheadPrefetch case TOfflineDisplayLocation.HomeTimelineRelatableAccounts => HomeTimelineRelatableAccounts case TOfflineDisplayLocation.NuxGeoCategory => NuxGeoCategory case TOfflineDisplayLocation.NuxInterestsCategory => NuxInterestsCategory case TOfflineDisplayLocation.TopArticles => TopArticles case TOfflineDisplayLocation.NuxPymkCategory => NuxPymkCategory case TOfflineDisplayLocation.HomeTimelineTweetRecs => HomeTimelineTweetRecs case TOfflineDisplayLocation.HtlBulkFriendFollows => HtlBulkFriendFollows case TOfflineDisplayLocation.NuxAutoFollow => NuxAutoFollow case TOfflineDisplayLocation.SearchBonusFollow => SearchBonusFollow case TOfflineDisplayLocation.ContentRecommender => ContentRecommender case TOfflineDisplayLocation.HomeTimelineReverseChron => HomeTimelineReverseChron case TOfflineDisplayLocation.EnumUnknownOfflineDisplayLocation(i) => throw new UnknownDisplayLocationException( s"Unknown offline display location thrift enum with value: ${i}") } } class UnknownDisplayLocationException(message: String) extends Exception(message) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/EngagementType.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.follow_recommendations.thriftscala.{EngagementType => TEngagementType} import com.twitter.follow_recommendations.logging.thriftscala.{ EngagementType => OfflineEngagementType } sealed trait EngagementType { def toThrift: TEngagementType def toOfflineThrift: OfflineEngagementType } object EngagementType { object Click extends EngagementType { override val toThrift: TEngagementType = TEngagementType.Click override val toOfflineThrift: OfflineEngagementType = OfflineEngagementType.Click } object Like extends EngagementType { override val toThrift: TEngagementType = TEngagementType.Like override val toOfflineThrift: OfflineEngagementType = OfflineEngagementType.Like } object Mention extends EngagementType { override val toThrift: TEngagementType = TEngagementType.Mention override val toOfflineThrift: OfflineEngagementType = OfflineEngagementType.Mention } object Retweet extends EngagementType { override val toThrift: TEngagementType = TEngagementType.Retweet override val toOfflineThrift: OfflineEngagementType = OfflineEngagementType.Retweet } object ProfileView extends EngagementType { override val toThrift: TEngagementType = TEngagementType.ProfileView override val toOfflineThrift: OfflineEngagementType = OfflineEngagementType.ProfileView } def fromThrift(engagementType: TEngagementType): EngagementType = engagementType match { case TEngagementType.Click => Click case TEngagementType.Like => Like case TEngagementType.Mention => Mention case TEngagementType.Retweet => Retweet case TEngagementType.ProfileView => ProfileView case TEngagementType.EnumUnknownEngagementType(i) => throw new UnknownEngagementTypeException( s"Unknown engagement type thrift enum with value: ${i}") } def fromOfflineThrift(engagementType: OfflineEngagementType): EngagementType = engagementType match { case OfflineEngagementType.Click => Click case OfflineEngagementType.Like => Like case OfflineEngagementType.Mention => Mention case OfflineEngagementType.Retweet => Retweet case OfflineEngagementType.ProfileView => ProfileView case OfflineEngagementType.EnumUnknownEngagementType(i) => throw new UnknownEngagementTypeException( s"Unknown engagement type offline thrift enum with value: ${i}") } } class UnknownEngagementTypeException(message: String) extends Exception(message) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FilterReason.scala ================================================ package com.twitter.follow_recommendations.common.models sealed trait FilterReason { def reason: String } object FilterReason { case object NoReason extends FilterReason { override val reason: String = "no_reason" } case class ParamReason(paramName: String) extends FilterReason { override val reason: String = s"param_$paramName" } case object ExcludedId extends FilterReason { override val reason: String = "excluded_id_from_request" } case object ProfileSidebarBlacklist extends FilterReason { override val reason: String = "profile_sidebar_blacklisted_id" } case object CuratedAccountsCompetitorList extends FilterReason { override val reason: String = "curated_blacklisted_id" } case class InvalidRelationshipTypes(relationshipTypes: String) extends FilterReason { override val reason: String = s"invalid_relationship_types $relationshipTypes" } case object ProfileId extends FilterReason { override val reason: String = "candidate_has_same_id_as_profile" } case object DismissedId extends FilterReason { override val reason: String = s"dismissed_candidate" } case object OptedOutId extends FilterReason { override val reason: String = s"candidate_opted_out_from_criteria_in_request" } // gizmoduck predicates case object NoUser extends FilterReason { override val reason: String = "no_user_result_from_gizmoduck" } case object AddressBookUndiscoverable extends FilterReason { override val reason: String = "not_discoverable_via_address_book" } case object PhoneBookUndiscoverable extends FilterReason { override val reason: String = "not_discoverable_via_phone_book" } case object Deactivated extends FilterReason { override val reason: String = "deactivated" } case object Suspended extends FilterReason { override val reason: String = "suspended" } case object Restricted extends FilterReason { override val reason: String = "restricted" } case object NsfwUser extends FilterReason { override val reason: String = "nsfwUser" } case object NsfwAdmin extends FilterReason { override val reason: String = "nsfwAdmin" } case object HssSignal extends FilterReason { override val reason: String = "hssSignal" } case object IsProtected extends FilterReason { override val reason: String = "isProtected" } case class CountryTakedown(countryCode: String) extends FilterReason { override val reason: String = s"takedown_in_$countryCode" } case object Blink extends FilterReason { override val reason: String = "blink" } case object AlreadyFollowed extends FilterReason { override val reason: String = "already_followed" } case object InvalidRelationship extends FilterReason { override val reason: String = "invalid_relationship" } case object NotFollowingTargetUser extends FilterReason { override val reason: String = "not_following_target_user" } case object CandidateSideHoldback extends FilterReason { override val reason: String = "candidate_side_holdback" } case object Inactive extends FilterReason { override val reason: String = "inactive" } case object MissingRecommendabilityData extends FilterReason { override val reason: String = "missing_recommendability_data" } case object HighTweetVelocity extends FilterReason { override val reason: String = "high_tweet_velocity" } case object AlreadyRecommended extends FilterReason { override val reason: String = "already_recommended" } case object MinStateNotMet extends FilterReason { override val reason: String = "min_state_user_not_met" } case object FailOpen extends FilterReason { override val reason: String = "fail_open" } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowContext.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.follow_recommendations.logging.{thriftscala => offline} import com.twitter.follow_recommendations.{thriftscala => t} case class FlowContext(steps: Seq[RecommendationStep]) { def toThrift: t.FlowContext = t.FlowContext(steps = steps.map(_.toThrift)) def toOfflineThrift: offline.OfflineFlowContext = offline.OfflineFlowContext(steps = steps.map(_.toOfflineThrift)) } object FlowContext { def fromThrift(flowContext: t.FlowContext): FlowContext = { FlowContext(steps = flowContext.steps.map(RecommendationStep.fromThrift)) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowRecommendation.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.follow_recommendations.logging.{thriftscala => offline} import com.twitter.follow_recommendations.{thriftscala => t} case class FlowRecommendation(userId: Long) { def toThrift: t.FlowRecommendation = t.FlowRecommendation(userId = userId) def toOfflineThrift: offline.OfflineFlowRecommendation = offline.OfflineFlowRecommendation(userId = userId) } object FlowRecommendation { def fromThrift(flowRecommendation: t.FlowRecommendation): FlowRecommendation = { FlowRecommendation( userId = flowRecommendation.userId ) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/GeohashAndCountryCode.scala ================================================ package com.twitter.follow_recommendations.common.models case class GeohashAndCountryCode(geohash: Option[String], countryCode: Option[String]) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasAdMetadata.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.adserver.{thriftscala => t} case class AdMetadata( insertPosition: Int, // use original ad impression info to avoid losing data in domain model translations adImpression: t.AdImpression) trait HasAdMetadata { def adMetadata: Option[AdMetadata] def adImpression: Option[t.AdImpression] = { adMetadata.map(_.adImpression) } def insertPosition: Option[Int] = { adMetadata.map(_.insertPosition) } def isPromotedAccount: Boolean = adMetadata.isDefined } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasByfSeedUserIds.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasByfSeedUserIds { def byfSeedUserIds: Option[Seq[Long]] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDataRecord.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.follow_recommendations.thriftscala.DebugDataRecord import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.util.Try import com.twitter.util.logging.Logging import scala.collection.convert.ImplicitConversions._ // contains the standard dataRecord struct, and the debug version if required case class RichDataRecord( dataRecord: Option[DataRecord] = None, debugDataRecord: Option[DebugDataRecord] = None, ) trait HasDataRecord extends Logging { def dataRecord: Option[RichDataRecord] def toDebugDataRecord(dr: DataRecord, featureContext: FeatureContext): DebugDataRecord = { val binaryFeatures: Option[Set[String]] = if (dr.isSetBinaryFeatures) { Some(dr.getBinaryFeatures.flatMap { id => Try(featureContext.getFeature(id).getFeatureName).toOption }.toSet) } else None val continuousFeatures: Option[Map[String, Double]] = if (dr.isSetContinuousFeatures) { Some(dr.getContinuousFeatures.flatMap { case (id, value) => Try(featureContext.getFeature(id).getFeatureName).toOption.map { id => id -> value.toDouble } }.toMap) } else None val discreteFeatures: Option[Map[String, Long]] = if (dr.isSetDiscreteFeatures) { Some(dr.getDiscreteFeatures.flatMap { case (id, value) => Try(featureContext.getFeature(id).getFeatureName).toOption.map { id => id -> value.toLong } }.toMap) } else None val stringFeatures: Option[Map[String, String]] = if (dr.isSetStringFeatures) { Some(dr.getStringFeatures.flatMap { case (id, value) => Try(featureContext.getFeature(id).getFeatureName).toOption.map { id => id -> value } }.toMap) } else None val sparseBinaryFeatures: Option[Map[String, Set[String]]] = if (dr.isSetSparseBinaryFeatures) { Some(dr.getSparseBinaryFeatures.flatMap { case (id, values) => Try(featureContext.getFeature(id).getFeatureName).toOption.map { id => id -> values.toSet } }.toMap) } else None val sparseContinuousFeatures: Option[Map[String, Map[String, Double]]] = if (dr.isSetSparseContinuousFeatures) { Some(dr.getSparseContinuousFeatures.flatMap { case (id, values) => Try(featureContext.getFeature(id).getFeatureName).toOption.map { id => id -> values.map { case (str, value) => str -> value.toDouble }.toMap } }.toMap) } else None DebugDataRecord( binaryFeatures = binaryFeatures, continuousFeatures = continuousFeatures, discreteFeatures = discreteFeatures, stringFeatures = stringFeatures, sparseBinaryFeatures = sparseBinaryFeatures, sparseContinuousFeatures = sparseContinuousFeatures, ) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDebugOptions.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.follow_recommendations.thriftscala.DebugParams case class DebugOptions( randomizationSeed: Option[Long] = None, fetchDebugInfo: Boolean = false, doNotLog: Boolean = false) object DebugOptions { def fromDebugParamsThrift(debugParams: DebugParams): DebugOptions = { DebugOptions( debugParams.randomizationSeed, debugParams.includeDebugInfoInResults.getOrElse(false), debugParams.doNotLog.getOrElse(false) ) } } trait HasDebugOptions { def debugOptions: Option[DebugOptions] def getRandomizationSeed: Option[Long] = debugOptions.flatMap(_.randomizationSeed) def fetchDebugInfo: Option[Boolean] = debugOptions.map(_.fetchDebugInfo) } trait HasFrsDebugOptions { def frsDebugOptions: Option[DebugOptions] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDismissedUserIds.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasDismissedUserIds { // user ids that are recently followed by the target user def dismissedUserIds: Option[Seq[Long]] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDisplayLocation.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasDisplayLocation { def displayLocation: DisplayLocation } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasEngagements.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasEngagements { def engagements: Seq[EngagementType] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasExcludedUserIds.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasExcludedUserIds { // user ids that are going to be excluded from recommendations def excludedUserIds: Seq[Long] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasGeohashAndCountryCode.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasGeohashAndCountryCode { def geohashAndCountryCode: Option[GeohashAndCountryCode] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInfoPerRankingStage.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasInfoPerRankingStage { def infoPerRankingStage: Option[scala.collection.Map[String, RankingInfo]] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInterestIds.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasCustomInterests { def customInterests: Option[Seq[String]] } trait HasUttInterests { def uttInterestIds: Option[Seq[Long]] } trait HasInterestIds extends HasCustomInterests with HasUttInterests {} ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInvalidRelationshipUserIds.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasInvalidRelationshipUserIds { // user ids that have invalid relationship with the target user def invalidRelationshipUserIds: Option[Set[Long]] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasIsSoftUser.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasIsSoftUser { def isSoftUser: Boolean } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasMutualFollowedUserIds.scala ================================================ package com.twitter.follow_recommendations.common.models // intersection of recent followers and followed by trait HasMutualFollowedUserIds extends HasRecentFollowedUserIds with HasRecentFollowedByUserIds { lazy val recentMutualFollows: Seq[Long] = recentFollowedUserIds.getOrElse(Nil).intersect(recentFollowedByUserIds.getOrElse(Nil)) lazy val numRecentMutualFollows: Int = recentMutualFollows.size } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasPreviousRecommendationsContext.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasPreviousRecommendationsContext { def previouslyRecommendedUserIDs: Set[Long] def previouslyFollowedUserIds: Set[Long] def skippedFollows: Set[Long] = { previouslyRecommendedUserIDs.diff(previouslyFollowedUserIds) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasProfileId.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasProfileId { def profileId: Long } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasQualityFactor.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasQualityFactor { def qualityFactor: Option[Double] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedByUserIds.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasRecentFollowedByUserIds { // user ids that have recently followed the target user; target user has been "followed by" them. def recentFollowedByUserIds: Option[Seq[Long]] lazy val numRecentFollowedByUserIds: Int = recentFollowedByUserIds.map(_.size).getOrElse(0) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIds.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasRecentFollowedUserIds { // user ids that are recently followed by the target user def recentFollowedUserIds: Option[Seq[Long]] // user ids that are recently followed by the target user in set data-structure lazy val recentFollowedUserIdsSet: Option[Set[Long]] = recentFollowedUserIds match { case Some(users) => Some(users.toSet) case None => Some(Set.empty) } lazy val numRecentFollowedUserIds: Int = recentFollowedUserIds.map(_.size).getOrElse(0) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIdsWithTime.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasRecentFollowedUserIdsWithTime { // user ids that are recently followed by the target user def recentFollowedUserIdsWithTime: Option[Seq[UserIdWithTimestamp]] lazy val numRecentFollowedUserIdsWithTime: Int = recentFollowedUserIdsWithTime.map(_.size).getOrElse(0) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentlyEngagedUserIds.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasRecentlyEngagedUserIds { val recentlyEngagedUserIds: Option[Seq[RecentlyEngagedUserId]] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecommendationFlowIdentifier.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasRecommendationFlowIdentifier { def recommendationFlowIdentifier: Option[String] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasScores.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasScores { def scores: Option[Scores] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasSimilarToContext.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasSimilarToContext { // user ids that are used to generate similar to recommendations def similarToUserIds: Seq[Long] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasTopicId.scala ================================================ package com.twitter.follow_recommendations.common.models trait HasTopicId { def topicId: Long } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserCandidateSourceDetails.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.hermit.ml.models.Feature import com.twitter.hermit.model.Algorithm import com.twitter.hermit.model.Algorithm.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier /** * Used to keep track of a candidate's source not so much as a feature but for filtering candidate * from specific sources (eg. GizmoduckPredicate) */ trait HasUserCandidateSourceDetails { candidateUser: CandidateUser => def userCandidateSourceDetails: Option[UserCandidateSourceDetails] def getAlgorithm: Algorithm = { val algorithm = for { details <- userCandidateSourceDetails identifier <- details.primaryCandidateSource algorithm <- Algorithm.withNameOpt(identifier.name) } yield algorithm algorithm.getOrElse(throw new Exception("Algorithm missing on candidate user!")) } def getAllAlgorithms: Seq[Algorithm] = { getCandidateSources.keys .flatMap(identifier => Algorithm.withNameOpt(identifier.name)).toSeq } def getAddressBookMetadata: Option[AddressBookMetadata] = { userCandidateSourceDetails.flatMap(_.addressBookMetadata) } def getCandidateSources: Map[CandidateSourceIdentifier, Option[Double]] = { userCandidateSourceDetails.map(_.candidateSourceScores).getOrElse(Map.empty) } def getCandidateRanks: Map[CandidateSourceIdentifier, Int] = { userCandidateSourceDetails.map(_.candidateSourceRanks).getOrElse(Map.empty) } def getCandidateFeatures: Map[CandidateSourceIdentifier, Seq[Feature]] = { userCandidateSourceDetails.map(_.candidateSourceFeatures).getOrElse(Map.empty) } def getPrimaryCandidateSource: Option[CandidateSourceIdentifier] = { userCandidateSourceDetails.flatMap(_.primaryCandidateSource) } def withCandidateSource(source: CandidateSourceIdentifier): CandidateUser = { withCandidateSourceAndScore(source, candidateUser.score) } def withCandidateSourceAndScore( source: CandidateSourceIdentifier, score: Option[Double] ): CandidateUser = { withCandidateSourceScoreAndFeatures(source, score, Nil) } def withCandidateSourceAndFeatures( source: CandidateSourceIdentifier, features: Seq[Feature] ): CandidateUser = { withCandidateSourceScoreAndFeatures(source, candidateUser.score, features) } def withCandidateSourceScoreAndFeatures( source: CandidateSourceIdentifier, score: Option[Double], features: Seq[Feature] ): CandidateUser = { val candidateSourceDetails = candidateUser.userCandidateSourceDetails .map { details => details.copy( primaryCandidateSource = Some(source), candidateSourceScores = details.candidateSourceScores + (source -> score), candidateSourceFeatures = details.candidateSourceFeatures + (source -> features) ) }.getOrElse( UserCandidateSourceDetails( Some(source), Map(source -> score), Map.empty, None, Map(source -> features))) candidateUser.copy( userCandidateSourceDetails = Some(candidateSourceDetails) ) } def addCandidateSourceScoresMap( scoreMap: Map[CandidateSourceIdentifier, Option[Double]] ): CandidateUser = { val candidateSourceDetails = candidateUser.userCandidateSourceDetails .map { details => details.copy(candidateSourceScores = details.candidateSourceScores ++ scoreMap) }.getOrElse(UserCandidateSourceDetails(scoreMap.keys.headOption, scoreMap, Map.empty, None)) candidateUser.copy( userCandidateSourceDetails = Some(candidateSourceDetails) ) } def addCandidateSourceRanksMap( rankMap: Map[CandidateSourceIdentifier, Int] ): CandidateUser = { val candidateSourceDetails = candidateUser.userCandidateSourceDetails .map { details => details.copy(candidateSourceRanks = details.candidateSourceRanks ++ rankMap) }.getOrElse(UserCandidateSourceDetails(rankMap.keys.headOption, Map.empty, rankMap, None)) candidateUser.copy( userCandidateSourceDetails = Some(candidateSourceDetails) ) } def addInfoPerRankingStage( rankingStage: String, scores: Option[Scores], rank: Int ): CandidateUser = { val scoresOpt: Option[Scores] = scores.orElse(candidateUser.scores) val originalInfoPerRankingStage = candidateUser.infoPerRankingStage.getOrElse(Map[String, RankingInfo]()) candidateUser.copy( infoPerRankingStage = Some(originalInfoPerRankingStage + (rankingStage -> RankingInfo(scoresOpt, Some(rank)))) ) } def addAddressBookMetadataIfAvailable( candidateSources: Seq[CandidateSourceIdentifier] ): CandidateUser = { val addressBookMetadata = AddressBookMetadata( inForwardPhoneBook = candidateSources.contains(AddressBookMetadata.ForwardPhoneBookCandidateSource), inReversePhoneBook = candidateSources.contains(AddressBookMetadata.ReversePhoneBookCandidateSource), inForwardEmailBook = candidateSources.contains(AddressBookMetadata.ForwardEmailBookCandidateSource), inReverseEmailBook = candidateSources.contains(AddressBookMetadata.ReverseEmailBookCandidateSource) ) val newCandidateSourceDetails = candidateUser.userCandidateSourceDetails .map { details => details.copy(addressBookMetadata = Some(addressBookMetadata)) }.getOrElse( UserCandidateSourceDetails( None, Map.empty, Map.empty, Some(addressBookMetadata), Map.empty)) candidateUser.copy( userCandidateSourceDetails = Some(newCandidateSourceDetails) ) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserState.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.core_workflows.user_model.thriftscala.UserState trait HasUserState { def userState: Option[UserState] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasWtfImpressions.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.util.Time trait HasWtfImpressions { def wtfImpressions: Option[Seq[WtfImpression]] lazy val numWtfImpressions: Int = wtfImpressions.map(_.size).getOrElse(0) lazy val candidateImpressions: Map[Long, WtfImpression] = wtfImpressions .map { imprMap => imprMap.map { i => i.candidateId -> i }.toMap }.getOrElse(Map.empty) lazy val latestImpressionTime: Time = { if (wtfImpressions.exists(_.nonEmpty)) { wtfImpressions.get.map(_.latestTime).max } else Time.Top } def getCandidateImpressionCounts(id: Long): Option[Int] = candidateImpressions.get(id).map(_.counts) def getCandidateLatestTime(id: Long): Option[Time] = { candidateImpressions.get(id).map(_.latestTime) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/OptimusRequest.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.timelines.configapi.HasParams /** Convenience trait to group together all traits needed for optimus ranking */ trait OptimusRequest extends HasParams with HasClientContext with HasDisplayLocation with HasInterestIds with HasDebugOptions with HasPreviousRecommendationsContext {} ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Product.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.product_mixer.core.model.common.identifier.ProductIdentifier import com.twitter.product_mixer.core.model.marshalling.request.{Product => ProductMixerProduct} object Product { case object MagicRecs extends ProductMixerProduct { override val identifier: ProductIdentifier = ProductIdentifier("MagicRecs") override val stringCenterProject: Option[String] = Some("people-discovery") } case object PlaceholderProductMixerProduct extends ProductMixerProduct { override val identifier: ProductIdentifier = ProductIdentifier("PlaceholderProductMixerProduct") } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RankingInfo.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.follow_recommendations.{thriftscala => t} import com.twitter.follow_recommendations.logging.{thriftscala => offline} case class RankingInfo( scores: Option[Scores], rank: Option[Int]) { def toThrift: t.RankingInfo = { t.RankingInfo(scores.map(_.toThrift), rank) } def toOfflineThrift: offline.RankingInfo = { offline.RankingInfo(scores.map(_.toOfflineThrift), rank) } } object RankingInfo { def fromThrift(rankingInfo: t.RankingInfo): RankingInfo = { RankingInfo( scores = rankingInfo.scores.map(Scores.fromThrift), rank = rankingInfo.rank ) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Reason.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.follow_recommendations.{thriftscala => t} import com.twitter.follow_recommendations.logging.{thriftscala => offline} case class FollowProof(followedBy: Seq[Long], numIds: Int) { def toThrift: t.FollowProof = { t.FollowProof(followedBy, numIds) } def toOfflineThrift: offline.FollowProof = offline.FollowProof(followedBy, numIds) } object FollowProof { def fromThrift(proof: t.FollowProof): FollowProof = { FollowProof(proof.userIds, proof.numIds) } } case class SimilarToProof(similarTo: Seq[Long]) { def toThrift: t.SimilarToProof = { t.SimilarToProof(similarTo) } def toOfflineThrift: offline.SimilarToProof = offline.SimilarToProof(similarTo) } object SimilarToProof { def fromThrift(proof: t.SimilarToProof): SimilarToProof = { SimilarToProof(proof.userIds) } } case class PopularInGeoProof(location: String) { def toThrift: t.PopularInGeoProof = { t.PopularInGeoProof(location) } def toOfflineThrift: offline.PopularInGeoProof = offline.PopularInGeoProof(location) } object PopularInGeoProof { def fromThrift(proof: t.PopularInGeoProof): PopularInGeoProof = { PopularInGeoProof(proof.location) } } case class TttInterestProof(interestId: Long, interestDisplayName: String) { def toThrift: t.TttInterestProof = { t.TttInterestProof(interestId, interestDisplayName) } def toOfflineThrift: offline.TttInterestProof = offline.TttInterestProof(interestId, interestDisplayName) } object TttInterestProof { def fromThrift(proof: t.TttInterestProof): TttInterestProof = { TttInterestProof(proof.interestId, proof.interestDisplayName) } } case class TopicProof(topicId: Long) { def toThrift: t.TopicProof = { t.TopicProof(topicId) } def toOfflineThrift: offline.TopicProof = offline.TopicProof(topicId) } object TopicProof { def fromThrift(proof: t.TopicProof): TopicProof = { TopicProof(proof.topicId) } } case class CustomInterest(query: String) { def toThrift: t.CustomInterestProof = { t.CustomInterestProof(query) } def toOfflineThrift: offline.CustomInterestProof = offline.CustomInterestProof(query) } object CustomInterest { def fromThrift(proof: t.CustomInterestProof): CustomInterest = { CustomInterest(proof.query) } } case class TweetsAuthorProof(tweetIds: Seq[Long]) { def toThrift: t.TweetsAuthorProof = { t.TweetsAuthorProof(tweetIds) } def toOfflineThrift: offline.TweetsAuthorProof = offline.TweetsAuthorProof(tweetIds) } object TweetsAuthorProof { def fromThrift(proof: t.TweetsAuthorProof): TweetsAuthorProof = { TweetsAuthorProof(proof.tweetIds) } } case class DeviceFollowProof(isDeviceFollow: Boolean) { def toThrift: t.DeviceFollowProof = { t.DeviceFollowProof(isDeviceFollow) } def toOfflineThrift: offline.DeviceFollowProof = offline.DeviceFollowProof(isDeviceFollow) } object DeviceFollowProof { def fromThrift(proof: t.DeviceFollowProof): DeviceFollowProof = { DeviceFollowProof(proof.isDeviceFollow) } } case class AccountProof( followProof: Option[FollowProof] = None, similarToProof: Option[SimilarToProof] = None, popularInGeoProof: Option[PopularInGeoProof] = None, tttInterestProof: Option[TttInterestProof] = None, topicProof: Option[TopicProof] = None, customInterestProof: Option[CustomInterest] = None, tweetsAuthorProof: Option[TweetsAuthorProof] = None, deviceFollowProof: Option[DeviceFollowProof] = None) { def toThrift: t.AccountProof = { t.AccountProof( followProof.map(_.toThrift), similarToProof.map(_.toThrift), popularInGeoProof.map(_.toThrift), tttInterestProof.map(_.toThrift), topicProof.map(_.toThrift), customInterestProof.map(_.toThrift), tweetsAuthorProof.map(_.toThrift), deviceFollowProof.map(_.toThrift) ) } def toOfflineThrift: offline.AccountProof = { offline.AccountProof( followProof.map(_.toOfflineThrift), similarToProof.map(_.toOfflineThrift), popularInGeoProof.map(_.toOfflineThrift), tttInterestProof.map(_.toOfflineThrift), topicProof.map(_.toOfflineThrift), customInterestProof.map(_.toOfflineThrift), tweetsAuthorProof.map(_.toOfflineThrift), deviceFollowProof.map(_.toOfflineThrift) ) } } object AccountProof { def fromThrift(proof: t.AccountProof): AccountProof = { AccountProof( proof.followProof.map(FollowProof.fromThrift), proof.similarToProof.map(SimilarToProof.fromThrift), proof.popularInGeoProof.map(PopularInGeoProof.fromThrift), proof.tttInterestProof.map(TttInterestProof.fromThrift), proof.topicProof.map(TopicProof.fromThrift), proof.customInterestProof.map(CustomInterest.fromThrift), proof.tweetsAuthorProof.map(TweetsAuthorProof.fromThrift), proof.deviceFollowProof.map(DeviceFollowProof.fromThrift) ) } } case class Reason(accountProof: Option[AccountProof]) { def toThrift: t.Reason = { t.Reason(accountProof.map(_.toThrift)) } def toOfflineThrift: offline.Reason = { offline.Reason(accountProof.map(_.toOfflineThrift)) } } object Reason { def fromThrift(reason: t.Reason): Reason = { Reason(reason.accountProof.map(AccountProof.fromThrift)) } } trait HasReason { def reason: Option[Reason] // helper methods below def followedBy: Option[Seq[Long]] = { for { reason <- reason accountProof <- reason.accountProof followProof <- accountProof.followProof } yield { followProof.followedBy } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecentlyEngagedUserId.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.follow_recommendations.logging.{thriftscala => offline} import com.twitter.follow_recommendations.{thriftscala => t} case class RecentlyEngagedUserId(id: Long, engagementType: EngagementType) { def toThrift: t.RecentlyEngagedUserId = t.RecentlyEngagedUserId(id = id, engagementType = engagementType.toThrift) def toOfflineThrift: offline.RecentlyEngagedUserId = offline.RecentlyEngagedUserId(id = id, engagementType = engagementType.toOfflineThrift) } object RecentlyEngagedUserId { def fromThrift(recentlyEngagedUserId: t.RecentlyEngagedUserId): RecentlyEngagedUserId = { RecentlyEngagedUserId( id = recentlyEngagedUserId.id, engagementType = EngagementType.fromThrift(recentlyEngagedUserId.engagementType) ) } def fromOfflineThrift( recentlyEngagedUserId: offline.RecentlyEngagedUserId ): RecentlyEngagedUserId = { RecentlyEngagedUserId( id = recentlyEngagedUserId.id, engagementType = EngagementType.fromOfflineThrift(recentlyEngagedUserId.engagementType) ) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecommendationStep.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.follow_recommendations.{thriftscala => t} import com.twitter.follow_recommendations.logging.{thriftscala => offline} case class RecommendationStep( recommendations: Seq[FlowRecommendation], followedUserIds: Set[Long]) { def toThrift: t.RecommendationStep = t.RecommendationStep( recommendations = recommendations.map(_.toThrift), followedUserIds = followedUserIds ) def toOfflineThrift: offline.OfflineRecommendationStep = offline.OfflineRecommendationStep( recommendations = recommendations.map(_.toOfflineThrift), followedUserIds = followedUserIds) } object RecommendationStep { def fromThrift(recommendationStep: t.RecommendationStep): RecommendationStep = { RecommendationStep( recommendations = recommendationStep.recommendations.map(FlowRecommendation.fromThrift), followedUserIds = recommendationStep.followedUserIds.toSet) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/STPGraph.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.hermit.model.Algorithm.Algorithm import com.twitter.wtf.scalding.jobs.strong_tie_prediction.FirstDegreeEdge import com.twitter.wtf.scalding.jobs.strong_tie_prediction.FirstDegreeEdgeInfo import com.twitter.wtf.scalding.jobs.strong_tie_prediction.SecondDegreeEdge case class PotentialFirstDegreeEdge( userId: Long, connectingId: Long, algorithm: Algorithm, score: Double, edgeInfo: FirstDegreeEdgeInfo) case class IntermediateSecondDegreeEdge( connectingId: Long, candidateId: Long, edgeInfo: FirstDegreeEdgeInfo) case class STPGraph( firstDegreeEdgeInfoList: List[FirstDegreeEdge], secondDegreeEdgeInfoList: List[SecondDegreeEdge]) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SafetyLevel.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.spam.rtf.thriftscala.{SafetyLevel => ThriftSafetyLevel} sealed trait SafetyLevel { def toThrift: ThriftSafetyLevel } object SafetyLevel { case object Recommendations extends SafetyLevel { override val toThrift = ThriftSafetyLevel.Recommendations } case object TopicsLandingPageTopicRecommendations extends SafetyLevel { override val toThrift = ThriftSafetyLevel.TopicsLandingPageTopicRecommendations } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Score.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.follow_recommendations.common.rankers.common.RankerId import com.twitter.follow_recommendations.common.rankers.common.RankerId.RankerId import com.twitter.follow_recommendations.logging.{thriftscala => offline} import com.twitter.follow_recommendations.{thriftscala => t} /** * Type of Score. This is used to differentiate scores. * * Define it as a trait so it is possible to add more information for different score types. */ sealed trait ScoreType { def getName: String } /** * Existing Score Types */ object ScoreType { /** * the score is calculated based on heuristics and most likely not normalized */ case object HeuristicBasedScore extends ScoreType { override def getName: String = "HeuristicBasedScore" } /** * probability of follow after the candidate is recommended to the user */ case object PFollowGivenReco extends ScoreType { override def getName: String = "PFollowGivenReco" } /** * probability of engage after the user follows the candidate */ case object PEngagementGivenFollow extends ScoreType { override def getName: String = "PEngagementGivenFollow" } /** * probability of engage per tweet impression */ case object PEngagementPerImpression extends ScoreType { override def getName: String = "PEngagementPerImpression" } /** * probability of engage per tweet impression */ case object PEngagementGivenReco extends ScoreType { override def getName: String = "PEngagementGivenReco" } def fromScoreTypeString(scoreTypeName: String): ScoreType = scoreTypeName match { case "HeuristicBasedScore" => HeuristicBasedScore case "PFollowGivenReco" => PFollowGivenReco case "PEngagementGivenFollow" => PEngagementGivenFollow case "PEngagementPerImpression" => PEngagementPerImpression case "PEngagementGivenReco" => PEngagementGivenReco } } /** * Represent the output from a certain ranker or scorer. All the fields are optional * * @param value value of the score * @param rankerId ranker id * @param scoreType score type */ final case class Score( value: Double, rankerId: Option[RankerId] = None, scoreType: Option[ScoreType] = None) { def toThrift: t.Score = t.Score( value = value, rankerId = rankerId.map(_.toString), scoreType = scoreType.map(_.getName) ) def toOfflineThrift: offline.Score = offline.Score( value = value, rankerId = rankerId.map(_.toString), scoreType = scoreType.map(_.getName) ) } object Score { val RandomScore = Score(0.0d, Some(RankerId.RandomRanker)) def optimusScore(score: Double, scoreType: ScoreType): Score = { Score(value = score, scoreType = Some(scoreType)) } def predictionScore(score: Double, rankerId: RankerId): Score = { Score(value = score, rankerId = Some(rankerId)) } def fromThrift(thriftScore: t.Score): Score = Score( value = thriftScore.value, rankerId = thriftScore.rankerId.flatMap(RankerId.getRankerByName), scoreType = thriftScore.scoreType.map(ScoreType.fromScoreTypeString) ) } /** * a list of scores */ final case class Scores( scores: Seq[Score], selectedRankerId: Option[RankerId] = None, isInProducerScoringExperiment: Boolean = false) { def toThrift: t.Scores = t.Scores( scores = scores.map(_.toThrift), selectedRankerId = selectedRankerId.map(_.toString), isInProducerScoringExperiment = isInProducerScoringExperiment ) def toOfflineThrift: offline.Scores = offline.Scores( scores = scores.map(_.toOfflineThrift), selectedRankerId = selectedRankerId.map(_.toString), isInProducerScoringExperiment = isInProducerScoringExperiment ) } object Scores { val Empty: Scores = Scores(Nil) def fromThrift(thriftScores: t.Scores): Scores = Scores( scores = thriftScores.scores.map(Score.fromThrift), selectedRankerId = thriftScores.selectedRankerId.flatMap(RankerId.getRankerByName), isInProducerScoringExperiment = thriftScores.isInProducerScoringExperiment ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Session.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.finagle.tracing.Trace object Session { /** * The sessionId in FRS is the finagle trace id which is static within the lifetime of a single * request. * * It is used when generating per-candidate tokens (in TrackingTokenTransform) and is also passed * in to downstream Optimus ranker requests. * */ def getSessionId: Long = Trace.id.traceId.toLong } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SignalData.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.simclusters_v2.thriftscala.InternalId import com.twitter.usersignalservice.thriftscala.SignalType import com.twitter.usersignalservice.thriftscala.Signal trait SignalData { val userId: Long val signalType: SignalType } case class RecentFollowsSignal( override val userId: Long, override val signalType: SignalType, followedUserId: Long, timestamp: Long) extends SignalData object RecentFollowsSignal { def fromUssSignal(targetUserId: Long, signal: Signal): RecentFollowsSignal = { val InternalId.UserId(followedUserId) = signal.targetInternalId.getOrElse( throw new IllegalArgumentException("RecentFollow Signal does not have internalId")) RecentFollowsSignal( userId = targetUserId, followedUserId = followedUserId, timestamp = signal.timestamp, signalType = signal.signalType ) } def getRecentFollowedUserIds( signalDataMap: Option[Map[SignalType, Seq[SignalData]]] ): Option[Seq[Long]] = { signalDataMap.map(_.getOrElse(SignalType.AccountFollow, default = Seq.empty).flatMap { case RecentFollowsSignal(userId, signalType, followedUserId, timestamp) => Some(followedUserId) case _ => None }) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TrackingToken.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.finagle.tracing.Trace import com.twitter.follow_recommendations.logging.{thriftscala => offline} import com.twitter.follow_recommendations.{thriftscala => t} import com.twitter.scrooge.BinaryThriftStructSerializer import com.twitter.suggests.controller_data.thriftscala.ControllerData import com.twitter.util.Base64StringEncoder /** * used for attribution per target-candidate pair * @param sessionId trace-id of the finagle request * @param controllerData 64-bit encoded binary attributes of our recommendation * @param algorithmId id for identifying a candidate source. maintained for backwards compatibility */ case class TrackingToken( sessionId: Long, displayLocation: Option[DisplayLocation], controllerData: Option[ControllerData], algorithmId: Option[Int]) { def toThrift: t.TrackingToken = { Trace.id.traceId.toLong t.TrackingToken( sessionId = sessionId, displayLocation = displayLocation.map(_.toThrift), controllerData = controllerData, algoId = algorithmId ) } def toOfflineThrift: offline.TrackingToken = { offline.TrackingToken( sessionId = sessionId, displayLocation = displayLocation.map(_.toOfflineThrift), controllerData = controllerData, algoId = algorithmId ) } } object TrackingToken { val binaryThriftSerializer = BinaryThriftStructSerializer[t.TrackingToken](t.TrackingToken) def serialize(trackingToken: TrackingToken): String = { Base64StringEncoder.encode(binaryThriftSerializer.toBytes(trackingToken.toThrift)) } def deserialize(trackingTokenStr: String): TrackingToken = { fromThrift(binaryThriftSerializer.fromBytes(Base64StringEncoder.decode(trackingTokenStr))) } def fromThrift(token: t.TrackingToken): TrackingToken = { TrackingToken( sessionId = token.sessionId, displayLocation = token.displayLocation.map(DisplayLocation.fromThrift), controllerData = token.controllerData, algorithmId = token.algoId ) } } trait HasTrackingToken { def trackingToken: Option[TrackingToken] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TweetCandidate.scala ================================================ package com.twitter.follow_recommendations.common.models case class TweetCandidate( tweetId: Long, authorId: Long, score: Option[Double]) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserCandidateSourceDetails.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.follow_recommendations.logging.{thriftscala => offline} import com.twitter.follow_recommendations.{thriftscala => t} import com.twitter.hermit.constants.AlgorithmFeedbackTokens._ import com.twitter.hermit.ml.models.Feature import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier /** * primaryCandidateSource param is showing the candidate source that responsible for generating this * candidate, as the candidate might have gone through multiple candidate sources to get generated * (for example if it has generated by a composite source). WeightedCandidateSourceRanker uses this * field to do the sampling over candidate sources. All the sources used for generating this * candidate (including the primary source) and their corresponding score exist in the * candidateSourceScores field. */ case class UserCandidateSourceDetails( primaryCandidateSource: Option[CandidateSourceIdentifier], candidateSourceScores: Map[CandidateSourceIdentifier, Option[Double]] = Map.empty, candidateSourceRanks: Map[CandidateSourceIdentifier, Int] = Map.empty, addressBookMetadata: Option[AddressBookMetadata] = None, candidateSourceFeatures: Map[CandidateSourceIdentifier, Seq[Feature]] = Map.empty, ) { def toThrift: t.CandidateSourceDetails = { t.CandidateSourceDetails( candidateSourceScores = Some(candidateSourceScores.map { case (identifier, score) => (identifier.name, score.getOrElse(0.0d)) }), primarySource = for { identifier <- primaryCandidateSource algo <- Algorithm.withNameOpt(identifier.name) feedbackToken <- AlgorithmToFeedbackTokenMap.get(algo) } yield feedbackToken ) } def toOfflineThrift: offline.CandidateSourceDetails = { offline.CandidateSourceDetails( candidateSourceScores = Some(candidateSourceScores.map { case (identifier, score) => (identifier.name, score.getOrElse(0.0d)) }), primarySource = for { identifier <- primaryCandidateSource algo <- Algorithm.withNameOpt(identifier.name) feedbackToken <- AlgorithmToFeedbackTokenMap.get(algo) } yield feedbackToken ) } } object UserCandidateSourceDetails { val algorithmNameMap: Map[String, Algorithm.Value] = Algorithm.values.map { algorithmValue: Algorithm.Value => (algorithmValue.toString, algorithmValue) }.toMap /** * This method is used to parse the candidate source of the candidates, which is only passed from * the scoreUserCandidates endpoint. We create custom candidate source identifiers which * CandidateAlgorithmSource will read from to hydrate the algorithm id feature. * candidateSourceScores will not be populated from the endpoint, but we add the conversion for * completeness. Note that the conversion uses the raw string of the Algorithm rather than the * assigned strings that we give to our own candidate sources in the FRS. */ def fromThrift(details: t.CandidateSourceDetails): UserCandidateSourceDetails = { val primaryCandidateSource: Option[CandidateSourceIdentifier] = for { primarySourceToken <- details.primarySource algo <- TokenToAlgorithmMap.get(primarySourceToken) } yield CandidateSourceIdentifier(algo.toString) val candidateSourceScores = for { scoreMap <- details.candidateSourceScores.toSeq (name, score) <- scoreMap algo <- algorithmNameMap.get(name) } yield { CandidateSourceIdentifier(algo.toString) -> Some(score) } val candidateSourceRanks = for { rankMap <- details.candidateSourceRanks.toSeq (name, rank) <- rankMap algo <- algorithmNameMap.get(name) } yield { CandidateSourceIdentifier(algo.toString) -> rank } UserCandidateSourceDetails( primaryCandidateSource = primaryCandidateSource, candidateSourceScores = candidateSourceScores.toMap, candidateSourceRanks = candidateSourceRanks.toMap, addressBookMetadata = None, candidateSourceFeatures = Map.empty ) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserIdAndTimestamp.scala ================================================ package com.twitter.follow_recommendations.common.models case class UserIdWithTimestamp(userId: Long, timeInMs: Long) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/WtfImpression.scala ================================================ package com.twitter.follow_recommendations.common.models import com.twitter.util.Time /** * Domain model for representing impressions on wtf recommendations in the past 16 days */ case class WtfImpression( candidateId: Long, displayLocation: DisplayLocation, latestTime: Time, counts: Int) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateParamPredicate.scala ================================================ package com.twitter.follow_recommendations.common.predicates import com.twitter.follow_recommendations.common.base.Predicate import com.twitter.follow_recommendations.common.base.PredicateResult import com.twitter.follow_recommendations.common.models.FilterReason import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.timelines.configapi.Param class CandidateParamPredicate[A <: HasParams]( param: Param[Boolean], reason: FilterReason) extends Predicate[A] { override def apply(candidate: A): Stitch[PredicateResult] = { if (candidate.params(param)) { Stitch.value(PredicateResult.Valid) } else { Stitch.value(PredicateResult.Invalid(Set(reason))) } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateSourceParamPredicate.scala ================================================ package com.twitter.follow_recommendations.common.predicates import com.twitter.follow_recommendations.common.base.Predicate import com.twitter.follow_recommendations.common.base.PredicateResult import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FilterReason import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.Param /** * This predicate allows us to filter candidates given its source. * To avoid bucket dilution, we only want to evaluate the param (which would implicitly trigger * bucketing for FSParams) only if the candidate source fn yields true. * The param provided should be true when we want to keep the candidate and false otherwise. */ class CandidateSourceParamPredicate( val param: Param[Boolean], val reason: FilterReason, candidateSources: Set[CandidateSourceIdentifier]) extends Predicate[CandidateUser] { override def apply(candidate: CandidateUser): Stitch[PredicateResult] = { // we want to avoid evaluating the param if the candidate source fn yields false if (candidate.getCandidateSources.keys.exists(candidateSources.contains) && !candidate.params( param)) { Stitch.value(PredicateResult.Invalid(Set(reason))) } else { Stitch.value(PredicateResult.Valid) } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CuratedCompetitorListPredicate.scala ================================================ package com.twitter.follow_recommendations.common.predicates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.Predicate import com.twitter.follow_recommendations.common.base.PredicateResult import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.follow_recommendations.common.models.FilterReason.CuratedAccountsCompetitorList import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import javax.inject.Inject import javax.inject.Singleton import com.twitter.conversions.DurationOps._ import com.twitter.escherbird.util.stitchcache.StitchCache @Singleton case class CuratedCompetitorListPredicate @Inject() ( statsReceiver: StatsReceiver, @Named(GuiceNamedConstants.CURATED_COMPETITOR_ACCOUNTS_FETCHER) competitorAccountFetcher: Fetcher[ String, Unit, Seq[Long] ]) extends Predicate[CandidateUser] { private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getName) private val cacheStats = stats.scope("cache") private val cache = StitchCache[String, Set[Long]]( maxCacheSize = CuratedCompetitorListPredicate.CacheNumberOfEntries, ttl = CuratedCompetitorListPredicate.CacheTTL, statsReceiver = cacheStats, underlyingCall = (competitorListPrefix: String) => query(competitorListPrefix) ) private def query(prefix: String): Stitch[Set[Long]] = competitorAccountFetcher.fetch(prefix).map(_.v.getOrElse(Nil).toSet) /** * Caveat here is that though the similarToUserIds allows for a Seq[Long], in practice we would * only return 1 userId. Multiple userId's would result in filtering candidates associated with * a different similarToUserId. For example: * - similarToUser1 -> candidate1, candidate2 * - similarToUser2 -> candidate3 * and in the competitorList store we have: * - similarToUser1 -> candidate3 * we'll be filtering candidate3 on account of similarToUser1, even though it was generated * with similarToUser2. This might still be desirable at a product level (since we don't want * to show these accounts anyway), but might not achieve what you intend to code-wise. */ override def apply(candidate: CandidateUser): Stitch[PredicateResult] = { cache.readThrough(CuratedCompetitorListPredicate.DefaultKey).map { competitorListAccounts => if (competitorListAccounts.contains(candidate.id)) { PredicateResult.Invalid(Set(CuratedAccountsCompetitorList)) } else { PredicateResult.Valid } } } } object CuratedCompetitorListPredicate { val DefaultKey: String = "default_list" val CacheTTL = 5.minutes val CacheNumberOfEntries = 5 } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/ExcludedUserIdPredicate.scala ================================================ package com.twitter.follow_recommendations.common.predicates import com.twitter.follow_recommendations.common.base.Predicate import com.twitter.follow_recommendations.common.base.PredicateResult import com.twitter.follow_recommendations.common.models.FilterReason.ExcludedId import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasExcludedUserIds import com.twitter.stitch.Stitch object ExcludedUserIdPredicate extends Predicate[(HasExcludedUserIds, CandidateUser)] { val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) val ExcludedStitch: Stitch[PredicateResult.Invalid] = Stitch.value(PredicateResult.Invalid(Set(ExcludedId))) override def apply(pair: (HasExcludedUserIds, CandidateUser)): Stitch[PredicateResult] = { val (excludedUserIds, candidate) = pair if (excludedUserIds.excludedUserIds.contains(candidate.id)) { ExcludedStitch } else { ValidStitch } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicate.scala ================================================ package com.twitter.follow_recommendations.common.predicates import com.google.inject.name.Named import com.twitter.core_workflows.user_model.thriftscala.UserState import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.Predicate import com.twitter.follow_recommendations.common.base.PredicateResult import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FilterReason import com.twitter.follow_recommendations.common.predicates.InactivePredicateParams._ import com.twitter.service.metastore.gen.thriftscala.UserRecommendabilityFeatures import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.timelines.configapi.HasParams import com.twitter.util.Duration import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton import com.twitter.conversions.DurationOps._ import com.twitter.escherbird.util.stitchcache.StitchCache import com.twitter.follow_recommendations.common.models.HasUserState import com.twitter.follow_recommendations.common.predicates.InactivePredicateParams.DefaultInactivityThreshold import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import java.lang.{Long => JLong} @Singleton case class InactivePredicate @Inject() ( statsReceiver: StatsReceiver, @Named(GuiceNamedConstants.USER_RECOMMENDABILITY_FETCHER) userRecommendabilityFetcher: Fetcher[ Long, Unit, UserRecommendabilityFeatures ]) extends Predicate[(HasParams with HasClientContext with HasUserState, CandidateUser)] { private val stats: StatsReceiver = statsReceiver.scope("InactivePredicate") private val cacheStats = stats.scope("cache") private def queryUserRecommendable(userId: Long): Stitch[Option[UserRecommendabilityFeatures]] = userRecommendabilityFetcher.fetch(userId).map(_.v) private val userRecommendableCache = StitchCache[JLong, Option[UserRecommendabilityFeatures]]( maxCacheSize = 100000, ttl = 12.hours, statsReceiver = cacheStats.scope("UserRecommendable"), underlyingCall = (userId: JLong) => queryUserRecommendable(userId) ) override def apply( targetAndCandidate: (HasParams with HasClientContext with HasUserState, CandidateUser) ): Stitch[PredicateResult] = { val (target, candidate) = targetAndCandidate userRecommendableCache .readThrough(candidate.id).map { case recFeaturesFetchResult => recFeaturesFetchResult match { case None => PredicateResult.Invalid(Set(FilterReason.MissingRecommendabilityData)) case Some(recFeatures) => if (disableInactivityPredicate(target, target.userState, recFeatures.userState)) { PredicateResult.Valid } else { val defaultInactivityThreshold = target.params(DefaultInactivityThreshold).days val hasBeenActiveRecently = recFeatures.lastStatusUpdateMs .map(Time.now - Time.fromMilliseconds(_)).getOrElse( Duration.Top) < defaultInactivityThreshold stats .scope(defaultInactivityThreshold.toString).counter( if (hasBeenActiveRecently) "active" else "inactive" ).incr() if (hasBeenActiveRecently && (!target .params(UseEggFilter) || recFeatures.isNotEgg.contains(1))) { PredicateResult.Valid } else { PredicateResult.Invalid(Set(FilterReason.Inactive)) } } } }.rescue { case e: Exception => stats.counter(e.getClass.getSimpleName).incr() Stitch(PredicateResult.Invalid(Set(FilterReason.FailOpen))) } } private[this] def disableInactivityPredicate( target: HasParams, consumerState: Option[UserState], candidateState: Option[UserState] ): Boolean = { target.params(MightBeDisabled) && consumerState.exists(InactivePredicate.ValidConsumerStates.contains) && ( ( candidateState.exists(InactivePredicate.ValidCandidateStates.contains) && !target.params(OnlyDisableForNewUserStateCandidates) ) || ( candidateState.contains(UserState.New) && target.params(OnlyDisableForNewUserStateCandidates) ) ) } } object InactivePredicate { val ValidConsumerStates: Set[UserState] = Set( UserState.HeavyNonTweeter, UserState.MediumNonTweeter, UserState.HeavyTweeter, UserState.MediumTweeter ) val ValidCandidateStates: Set[UserState] = Set(UserState.New, UserState.VeryLight, UserState.Light, UserState.NearZero) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicateParams.scala ================================================ package com.twitter.follow_recommendations.common.predicates import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.Param object InactivePredicateParams { case object DefaultInactivityThreshold extends FSBoundedParam[Int]( name = "inactive_predicate_default_inactivity_threshold", default = 60, min = 1, max = 500 ) case object UseEggFilter extends Param[Boolean](true) case object MightBeDisabled extends FSParam[Boolean]("inactive_predicate_might_be_disabled", true) case object OnlyDisableForNewUserStateCandidates extends FSParam[Boolean]( "inactive_predicate_only_disable_for_new_user_state_candidates", false) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/PreviouslyRecommendedUserIdsPredicate.scala ================================================ package com.twitter.follow_recommendations.common.predicates import com.twitter.follow_recommendations.common.base.Predicate import com.twitter.follow_recommendations.common.base.PredicateResult import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FilterReason import com.twitter.follow_recommendations.common.models.HasPreviousRecommendationsContext import com.twitter.stitch.Stitch import javax.inject.Singleton @Singleton class PreviouslyRecommendedUserIdsPredicate extends Predicate[(HasPreviousRecommendationsContext, CandidateUser)] { override def apply( pair: (HasPreviousRecommendationsContext, CandidateUser) ): Stitch[PredicateResult] = { val (targetUser, candidate) = pair val previouslyRecommendedUserIDs = targetUser.previouslyRecommendedUserIDs if (!previouslyRecommendedUserIDs.contains(candidate.id)) { PreviouslyRecommendedUserIdsPredicate.ValidStitch } else { PreviouslyRecommendedUserIdsPredicate.AlreadyRecommendedStitch } } } object PreviouslyRecommendedUserIdsPredicate { val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) val AlreadyRecommendedStitch: Stitch[PredicateResult.Invalid] = Stitch.value(PredicateResult.Invalid(Set(FilterReason.AlreadyRecommended))) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicate.scala ================================================ package com.twitter.follow_recommendations.common.predicates.dismiss import com.twitter.follow_recommendations.common.base.Predicate import com.twitter.follow_recommendations.common.base.PredicateResult import com.twitter.follow_recommendations.common.models.FilterReason.DismissedId import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDismissedUserIds import com.twitter.stitch.Stitch import javax.inject.Singleton @Singleton class DismissedCandidatePredicate extends Predicate[(HasDismissedUserIds, CandidateUser)] { override def apply(pair: (HasDismissedUserIds, CandidateUser)): Stitch[PredicateResult] = { val (targetUser, candidate) = pair targetUser.dismissedUserIds .map { dismissedUserIds => if (!dismissedUserIds.contains(candidate.id)) { DismissedCandidatePredicate.ValidStitch } else { DismissedCandidatePredicate.DismissedStitch } }.getOrElse(DismissedCandidatePredicate.ValidStitch) } } object DismissedCandidatePredicate { val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) val DismissedStitch: Stitch[PredicateResult.Invalid] = Stitch.value(PredicateResult.Invalid(Set(DismissedId))) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicateParams.scala ================================================ package com.twitter.follow_recommendations.common.predicates.dismiss import com.twitter.conversions.DurationOps._ import com.twitter.timelines.configapi.Param import com.twitter.util.Duration object DismissedCandidatePredicateParams { case object LookBackDuration extends Param[Duration](180.days) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", "stitch/stitch-gizmoduck", "util/util-slf4j-api/src/main/scala", "util/util-thrift", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicate.scala ================================================ package com.twitter.follow_recommendations.common.predicates.gizmoduck import com.twitter.decider.Decider import com.twitter.decider.RandomRecipient import com.twitter.escherbird.util.stitchcache.StitchCache import com.twitter.finagle.Memcached.Client import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.util.DefaultTimer import com.twitter.follow_recommendations.common.base.StatsUtil import com.twitter.follow_recommendations.common.base.Predicate import com.twitter.follow_recommendations.common.base.PredicateResult import com.twitter.follow_recommendations.common.clients.cache.MemcacheClient import com.twitter.follow_recommendations.common.clients.cache.ThriftBijection import com.twitter.follow_recommendations.common.models.FilterReason._ import com.twitter.follow_recommendations.common.models.AddressBookMetadata import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FilterReason import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicate._ import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicateParams._ import com.twitter.follow_recommendations.configapi.deciders.DeciderKey import com.twitter.gizmoduck.thriftscala.LabelValue.BlinkBad import com.twitter.gizmoduck.thriftscala.LabelValue.BlinkWorst import com.twitter.gizmoduck.thriftscala.LabelValue import com.twitter.gizmoduck.thriftscala.LookupContext import com.twitter.gizmoduck.thriftscala.QueryFields import com.twitter.gizmoduck.thriftscala.User import com.twitter.gizmoduck.thriftscala.UserResult import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.scrooge.CompactThriftSerializer import com.twitter.spam.rtf.thriftscala.SafetyLevel import com.twitter.stitch.Stitch import com.twitter.stitch.gizmoduck.Gizmoduck import com.twitter.timelines.configapi.HasParams import com.twitter.util.Duration import com.twitter.util.logging.Logging import java.lang.{Long => JLong} import javax.inject.Inject import javax.inject.Singleton /** * In this filter, we want to check 4 categories of conditions: * - if candidate is discoverable given that it's from an address-book/phone-book based source * - if candidate is unsuitable based on it's safety sub-fields in gizmoduck * - if candidate is withheld because of country-specific take-down policies * - if candidate is marked as bad/worst based on blink labels * We fail close on the query as this is a product-critical filter */ @Singleton case class GizmoduckPredicate @Inject() ( gizmoduck: Gizmoduck, client: Client, statsReceiver: StatsReceiver, decider: Decider = Decider.False) extends Predicate[(HasClientContext with HasParams, CandidateUser)] with Logging { private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getName) // track # of Gizmoduck predicate queries that yielded valid & invalid predicate results private val validPredicateResultCounter = stats.counter("predicate_valid") private val invalidPredicateResultCounter = stats.counter("predicate_invalid") // track # of cases where no Gizmoduck user was found private val noGizmoduckUserCounter = stats.counter("no_gizmoduck_user_found") private val gizmoduckCache = StitchCache[JLong, UserResult]( maxCacheSize = MaxCacheSize, ttl = CacheTTL, statsReceiver = stats.scope("cache"), underlyingCall = getByUserId ) // Distributed Twemcache to store UserResult objects keyed on user IDs val bijection = new ThriftBijection[UserResult] { override val serializer = CompactThriftSerializer(UserResult) } val memcacheClient = MemcacheClient[UserResult]( client = client, dest = "/s/cache/frs:twemcaches", valueBijection = bijection, ttl = CacheTTL, statsReceiver = stats.scope("twemcache") ) // main method used to apply GizmoduckPredicate to a candidate user override def apply( pair: (HasClientContext with HasParams, CandidateUser) ): Stitch[PredicateResult] = { val (request, candidate) = pair // measure the latency of the getGizmoduckPredicateResult, since this predicate // check is product-critical and relies on querying a core service (Gizmoduck) StatsUtil.profileStitch( getGizmoduckPredicateResult(request, candidate), stats.scope("getGizmoduckPredicateResult") ) } private def getGizmoduckPredicateResult( request: HasClientContext with HasParams, candidate: CandidateUser ): Stitch[PredicateResult] = { val timeout: Duration = request.params(GizmoduckGetTimeout) val deciderKey: String = DeciderKey.EnableGizmoduckCaching.toString val enableDistributedCaching: Boolean = decider.isAvailable(deciderKey, Some(RandomRecipient)) // try getting an existing UserResult from cache if possible val userResultStitch: Stitch[UserResult] = enableDistributedCaching match { // read from memcache case true => memcacheClient.readThrough( // add a key prefix to address cache key collisions key = "GizmoduckPredicate" + candidate.id.toString, underlyingCall = () => getByUserId(candidate.id) ) // read from local cache case false => gizmoduckCache.readThrough(candidate.id) } val predicateResultStitch = userResultStitch.map { userResult => { val predicateResult = getPredicateResult(request, candidate, userResult) if (enableDistributedCaching) { predicateResult match { case PredicateResult.Valid => stats.scope("twemcache").counter("predicate_valid").incr() case PredicateResult.Invalid(reasons) => stats.scope("twemcache").counter("predicate_invalid").incr() } // log metrics to check if local cache value matches distributed cache value logPredicateResultEquality( request, candidate, predicateResult ) } else { predicateResult match { case PredicateResult.Valid => stats.scope("cache").counter("predicate_valid").incr() case PredicateResult.Invalid(reasons) => stats.scope("cache").counter("predicate_invalid").incr() } } predicateResult } } predicateResultStitch .within(timeout)(DefaultTimer) .rescue { // fail-open when timeout or exception case e: Exception => stats.scope("rescued").counter(e.getClass.getSimpleName).incr() invalidPredicateResultCounter.incr() Stitch(PredicateResult.Invalid(Set(FailOpen))) } } private def logPredicateResultEquality( request: HasClientContext with HasParams, candidate: CandidateUser, predicateResult: PredicateResult ): Unit = { val localCachedUserResult = Option(gizmoduckCache.cache.getIfPresent(candidate.id)) if (localCachedUserResult.isDefined) { val localPredicateResult = getPredicateResult(request, candidate, localCachedUserResult.get) localPredicateResult.equals(predicateResult) match { case true => stats.scope("has_equal_predicate_value").counter("true").incr() case false => stats.scope("has_equal_predicate_value").counter("false").incr() } } else { stats.scope("has_equal_predicate_value").counter("undefined").incr() } } // method to get PredicateResult from UserResult def getPredicateResult( request: HasClientContext with HasParams, candidate: CandidateUser, userResult: UserResult, ): PredicateResult = { userResult.user match { case Some(user) => val abPbReasons = getAbPbReason(user, candidate.getAddressBookMetadata) val safetyReasons = getSafetyReasons(user) val countryTakedownReasons = getCountryTakedownReasons(user, request.getCountryCode) val blinkReasons = getBlinkReasons(user) val allReasons = abPbReasons ++ safetyReasons ++ countryTakedownReasons ++ blinkReasons if (allReasons.nonEmpty) { invalidPredicateResultCounter.incr() PredicateResult.Invalid(allReasons) } else { validPredicateResultCounter.incr() PredicateResult.Valid } case None => noGizmoduckUserCounter.incr() invalidPredicateResultCounter.incr() PredicateResult.Invalid(Set(NoUser)) } } private def getByUserId(userId: JLong): Stitch[UserResult] = { StatsUtil.profileStitch( gizmoduck.getById(userId = userId, queryFields = queryFields, context = lookupContext), stats.scope("getByUserId") ) } } object GizmoduckPredicate { private[gizmoduck] val lookupContext: LookupContext = LookupContext(`includeDeactivated` = true, `safetyLevel` = Some(SafetyLevel.Recommendations)) private[gizmoduck] val queryFields: Set[QueryFields] = Set( QueryFields.Discoverability, // needed for Address Book / Phone Book discoverability checks in getAbPbReason QueryFields.Safety, // needed for user state safety checks in getSafetyReasons, getCountryTakedownReasons QueryFields.Labels, // needed for user label checks in getBlinkReasons QueryFields.Takedowns // needed for checking takedown labels for a user in getCountryTakedownReasons ) private[gizmoduck] val BlinkLabels: Set[LabelValue] = Set(BlinkBad, BlinkWorst) private[gizmoduck] def getAbPbReason( user: User, abMetadataOpt: Option[AddressBookMetadata] ): Set[FilterReason] = { (for { discoverability <- user.discoverability abMetadata <- abMetadataOpt } yield { val AddressBookMetadata(fwdPb, rvPb, fwdAb, rvAb) = abMetadata val abReason: Set[FilterReason] = if ((!discoverability.discoverableByEmail) && (fwdAb || rvAb)) Set(AddressBookUndiscoverable) else Set.empty val pbReason: Set[FilterReason] = if ((!discoverability.discoverableByMobilePhone) && (fwdPb || rvPb)) Set(PhoneBookUndiscoverable) else Set.empty abReason ++ pbReason }).getOrElse(Set.empty) } private[gizmoduck] def getSafetyReasons(user: User): Set[FilterReason] = { user.safety .map { s => val deactivatedReason: Set[FilterReason] = if (s.deactivated) Set(Deactivated) else Set.empty val suspendedReason: Set[FilterReason] = if (s.suspended) Set(Suspended) else Set.empty val restrictedReason: Set[FilterReason] = if (s.restricted) Set(Restricted) else Set.empty val nsfwUserReason: Set[FilterReason] = if (s.nsfwUser) Set(NsfwUser) else Set.empty val nsfwAdminReason: Set[FilterReason] = if (s.nsfwAdmin) Set(NsfwAdmin) else Set.empty val isProtectedReason: Set[FilterReason] = if (s.isProtected) Set(IsProtected) else Set.empty deactivatedReason ++ suspendedReason ++ restrictedReason ++ nsfwUserReason ++ nsfwAdminReason ++ isProtectedReason }.getOrElse(Set.empty) } private[gizmoduck] def getCountryTakedownReasons( user: User, countryCodeOpt: Option[String] ): Set[FilterReason] = { (for { safety <- user.safety.toSeq if safety.hasTakedown takedowns <- user.takedowns.toSeq takedownCountry <- takedowns.countryCodes requestingCountry <- countryCodeOpt if takedownCountry.toLowerCase == requestingCountry.toLowerCase } yield Set(CountryTakedown(takedownCountry.toLowerCase))).flatten.toSet } private[gizmoduck] def getBlinkReasons(user: User): Set[FilterReason] = { user.labels .map(_.labels.map(_.labelValue)) .getOrElse(Nil) .exists(BlinkLabels.contains) for { labels <- user.labels.toSeq label <- labels.labels if (BlinkLabels.contains(label.labelValue)) } yield Set(Blink) }.flatten.toSet } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateCache.scala ================================================ package com.twitter.follow_recommendations.common.predicates.gizmoduck import java.util.concurrent.TimeUnit import com.google.common.base.Ticker import com.google.common.cache.CacheBuilder import com.google.common.cache.Cache import com.twitter.finagle.stats.StatsReceiver import com.twitter.util.Time import com.twitter.util.Duration /** * In-memory cache used for caching GizmoduckPredicate query calls in * com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicate. * * References the cache implementation in com.twitter.escherbird.util.stitchcache, * but without the underlying Stitch call. */ object GizmoduckPredicateCache { private[GizmoduckPredicateCache] class TimeTicker extends Ticker { override def read(): Long = Time.now.inNanoseconds } def apply[K, V]( maxCacheSize: Int, ttl: Duration, statsReceiver: StatsReceiver ): Cache[K, V] = { val cache: Cache[K, V] = CacheBuilder .newBuilder() .maximumSize(maxCacheSize) .asInstanceOf[CacheBuilder[K, V]] .expireAfterWrite(ttl.inSeconds, TimeUnit.SECONDS) .recordStats() .ticker(new TimeTicker()) .build() // metrics for tracking cache usage statsReceiver.provideGauge("cache_size") { cache.size.toFloat } statsReceiver.provideGauge("cache_hits") { cache.stats.hitCount.toFloat } statsReceiver.provideGauge("cache_misses") { cache.stats.missCount.toFloat } statsReceiver.provideGauge("cache_hit_rate") { cache.stats.hitRate.toFloat } statsReceiver.provideGauge("cache_evictions") { cache.stats.evictionCount.toFloat } cache } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.predicates.gizmoduck import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicateParams._ import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.util.Duration import javax.inject.Inject import javax.inject.Singleton @Singleton class GizmoduckPredicateFSConfig @Inject() () extends FeatureSwitchConfig { override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( GizmoduckGetTimeout ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateParams.scala ================================================ package com.twitter.follow_recommendations.common.predicates.gizmoduck import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.util.Duration import com.twitter.conversions.DurationOps._ object GizmoduckPredicateParams { case object GizmoduckGetTimeout extends FSBoundedParam[Duration]( name = "gizmoduck_predicate_timeout_in_millis", default = 200.millisecond, min = 1.millisecond, max = 500.millisecond) with HasDurationConversion { override def durationConversion: DurationConversion = DurationConversion.FromMillis } val MaxCacheSize: Int = 250000 val CacheTTL: Duration = Duration.fromHours(6) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "strato/config/columns/hss/user_signals/api:api-strato-client", "util/util-slf4j-api/src/main/scala", "util/util-thrift", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicate.scala ================================================ package com.twitter.follow_recommendations.common.predicates.hss import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.util.DefaultTimer import com.twitter.follow_recommendations.common.base.Predicate import com.twitter.follow_recommendations.common.base.PredicateResult import com.twitter.follow_recommendations.common.base.StatsUtil import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FilterReason import com.twitter.follow_recommendations.common.models.FilterReason.FailOpen import com.twitter.hss.api.thriftscala.SignalValue import com.twitter.hss.api.thriftscala.UserHealthSignal.AgathaCseDouble import com.twitter.hss.api.thriftscala.UserHealthSignal.NsfwAgathaUserScoreDouble import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.hss.user_signals.api.HealthSignalsOnUserClientColumn import com.twitter.timelines.configapi.HasParams import com.twitter.util.logging.Logging import com.twitter.util.Duration import javax.inject.Inject import javax.inject.Singleton /** * Filter out candidates based on Health Signal Store (HSS) health signals */ @Singleton case class HssPredicate @Inject() ( healthSignalsOnUserClientColumn: HealthSignalsOnUserClientColumn, statsReceiver: StatsReceiver) extends Predicate[(HasClientContext with HasParams, CandidateUser)] with Logging { private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getName) override def apply( pair: (HasClientContext with HasParams, CandidateUser) ): Stitch[PredicateResult] = { val (request, candidate) = pair StatsUtil.profileStitch( getHssPredicateResult(request, candidate), stats.scope("getHssPredicateResult") ) } private def getHssPredicateResult( request: HasClientContext with HasParams, candidate: CandidateUser ): Stitch[PredicateResult] = { val hssCseScoreThreshold: Double = request.params(HssPredicateParams.HssCseScoreThreshold) val hssNsfwScoreThreshold: Double = request.params(HssPredicateParams.HssNsfwScoreThreshold) val timeout: Duration = request.params(HssPredicateParams.HssApiTimeout) healthSignalsOnUserClientColumn.fetcher .fetch(candidate.id, Seq(AgathaCseDouble, NsfwAgathaUserScoreDouble)) .map { fetchResult => fetchResult.v match { case Some(response) => val agathaCseScoreDouble: Double = userHealthSignalValueToDoubleOpt( response.signalValues.get(AgathaCseDouble)).getOrElse(0d) val agathaNsfwScoreDouble: Double = userHealthSignalValueToDoubleOpt( response.signalValues.get(NsfwAgathaUserScoreDouble)).getOrElse(0d) stats.stat("agathaCseScoreDistribution").add(agathaCseScoreDouble.toFloat) stats.stat("agathaNsfwScoreDistribution").add(agathaNsfwScoreDouble.toFloat) /** * Only filter out the candidate when it has both high Agatha CSE score and NSFW score, as the Agatha CSE * model is an old one that may not be precise or have high recall. */ if (agathaCseScoreDouble >= hssCseScoreThreshold && agathaNsfwScoreDouble >= hssNsfwScoreThreshold) { PredicateResult.Invalid(Set(FilterReason.HssSignal)) } else { PredicateResult.Valid } case None => PredicateResult.Valid } } .within(timeout)(DefaultTimer) .rescue { case e: Exception => stats.scope("rescued").counter(e.getClass.getSimpleName).incr() Stitch(PredicateResult.Invalid(Set(FailOpen))) } } private def userHealthSignalValueToDoubleOpt(signalValue: Option[SignalValue]): Option[Double] = { signalValue match { case Some(SignalValue.DoubleValue(value)) => Some(value) case _ => None } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.predicates.hss import com.twitter.follow_recommendations.common.predicates.hss.HssPredicateParams._ import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.util.Duration import javax.inject.Inject import javax.inject.Singleton @Singleton class HssPredicateFSConfig @Inject() () extends FeatureSwitchConfig { override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( HssCseScoreThreshold, HssNsfwScoreThreshold, ) override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( HssApiTimeout ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateParams.scala ================================================ package com.twitter.follow_recommendations.common.predicates.hss import com.twitter.conversions.DurationOps._ import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.util.Duration object HssPredicateParams { object HssCseScoreThreshold extends FSBoundedParam[Double]( "hss_predicate_cse_score_threshold", default = 0.992d, min = 0.0d, max = 1.0d) object HssNsfwScoreThreshold extends FSBoundedParam[Double]( "hss_predicate_nsfw_score_threshold", default = 1.5d, min = -100.0d, max = 100.0d) object HssApiTimeout extends FSBoundedParam[Duration]( name = "hss_predicate_timeout_in_millis", default = 200.millisecond, min = 1.millisecond, max = 500.millisecond) with HasDurationConversion { override def durationConversion: DurationConversion = DurationConversion.FromMillis } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "src/thrift/com/twitter/socialgraph:thrift-scala", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/InvalidRelationshipPredicate.scala ================================================ package com.twitter.follow_recommendations.common.predicates.sgs import com.twitter.follow_recommendations.common.base.Predicate import com.twitter.follow_recommendations.common.base.PredicateResult import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FilterReason import com.twitter.follow_recommendations.common.models.HasInvalidRelationshipUserIds import com.twitter.stitch.Stitch import javax.inject.Singleton @Singleton class InvalidRelationshipPredicate extends Predicate[(HasInvalidRelationshipUserIds, CandidateUser)] { override def apply( pair: (HasInvalidRelationshipUserIds, CandidateUser) ): Stitch[PredicateResult] = { val (targetUser, candidate) = pair targetUser.invalidRelationshipUserIds match { case Some(users) => if (!users.contains(candidate.id)) { InvalidRelationshipPredicate.ValidStitch } else { Stitch.value(InvalidRelationshipPredicate.InvalidRelationshipStitch) } case None => Stitch.value(PredicateResult.Valid) } } } object InvalidRelationshipPredicate { val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) val InvalidRelationshipStitch: PredicateResult.Invalid = PredicateResult.Invalid(Set(FilterReason.InvalidRelationship)) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/RecentFollowingPredicate.scala ================================================ package com.twitter.follow_recommendations.common.predicates.sgs import com.twitter.follow_recommendations.common.base.Predicate import com.twitter.follow_recommendations.common.base.PredicateResult import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FilterReason import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds import com.twitter.stitch.Stitch import javax.inject.Singleton @Singleton class RecentFollowingPredicate extends Predicate[(HasRecentFollowedUserIds, CandidateUser)] { override def apply(pair: (HasRecentFollowedUserIds, CandidateUser)): Stitch[PredicateResult] = { val (targetUser, candidate) = pair targetUser.recentFollowedUserIdsSet match { case Some(users) => if (!users.contains(candidate.id)) { RecentFollowingPredicate.ValidStitch } else { RecentFollowingPredicate.AlreadyFollowedStitch } case None => RecentFollowingPredicate.ValidStitch } } } object RecentFollowingPredicate { val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) val AlreadyFollowedStitch: Stitch[PredicateResult.Invalid] = Stitch.value(PredicateResult.Invalid(Set(FilterReason.AlreadyFollowed))) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.predicates.sgs import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.util.Duration import javax.inject.Inject import javax.inject.Singleton @Singleton class SgsPredicateFSConfig @Inject() () extends FeatureSwitchConfig { override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( SgsPredicateParams.SgsRelationshipsPredicateTimeout ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateParams.scala ================================================ package com.twitter.follow_recommendations.common.predicates.sgs import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.util.Duration import com.twitter.conversions.DurationOps._ object SgsPredicateParams { case object SgsRelationshipsPredicateTimeout extends FSBoundedParam[Duration]( name = "sgs_predicate_relationships_timeout_in_millis", default = 300.millisecond, min = 1.millisecond, max = 1000.millisecond) with HasDurationConversion { override def durationConversion: DurationConversion = DurationConversion.FromMillis } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsByUserIdPredicate.scala ================================================ package com.twitter.follow_recommendations.common.predicates.sgs import com.google.common.annotations.VisibleForTesting import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.Predicate import com.twitter.follow_recommendations.common.base.PredicateResult import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FilterReason.InvalidRelationshipTypes import com.twitter.socialgraph.thriftscala.ExistsRequest import com.twitter.socialgraph.thriftscala.ExistsResult import com.twitter.socialgraph.thriftscala.LookupContext import com.twitter.socialgraph.thriftscala.Relationship import com.twitter.socialgraph.thriftscala.RelationshipType import com.twitter.stitch.Stitch import com.twitter.stitch.socialgraph.SocialGraph import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton class SgsRelationshipsByUserIdPredicate( socialGraph: SocialGraph, relationshipMappings: Seq[RelationshipMapping], statsReceiver: StatsReceiver) extends Predicate[(Option[Long], CandidateUser)] with Logging { private val InvalidFromPrimaryCandidateSourceName = "invalid_from_primary_candidate_source" private val InvalidFromCandidateSourceName = "invalid_from_candidate_source" private val NoPrimaryCandidateSource = "no_primary_candidate_source" private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getName) override def apply( pair: (Option[Long], CandidateUser) ): Stitch[PredicateResult] = { val (idOpt, candidate) = pair val relationships = relationshipMappings.map { relationshipMapping: RelationshipMapping => Relationship( relationshipMapping.relationshipType, relationshipMapping.includeBasedOnRelationship) } idOpt .map { id: Long => val existsRequest = ExistsRequest( id, candidate.id, relationships = relationships, context = SgsRelationshipsByUserIdPredicate.UnionLookupContext ) socialGraph .exists(existsRequest).map { existsResult: ExistsResult => if (existsResult.exists) { candidate.getPrimaryCandidateSource match { case Some(candidateSource) => stats .scope(InvalidFromPrimaryCandidateSourceName).counter( candidateSource.name).incr() case None => stats .scope(InvalidFromPrimaryCandidateSourceName).counter( NoPrimaryCandidateSource).incr() } candidate.getCandidateSources.foreach({ case (candidateSource, _) => stats .scope(InvalidFromCandidateSourceName).counter(candidateSource.name).incr() }) PredicateResult.Invalid(Set(InvalidRelationshipTypes(relationshipMappings .map { relationshipMapping: RelationshipMapping => relationshipMapping.relationshipType }.mkString(", ")))) } else { PredicateResult.Valid } } } // if no user id is present, return true by default .getOrElse(Stitch.value(PredicateResult.Valid)) } } object SgsRelationshipsByUserIdPredicate { // OR Operation @VisibleForTesting private[follow_recommendations] val UnionLookupContext = Some( LookupContext(performUnion = Some(true))) } @Singleton class ExcludeNonFollowersSgsPredicate @Inject() ( socialGraph: SocialGraph, statsReceiver: StatsReceiver) extends SgsRelationshipsByUserIdPredicate( socialGraph, Seq(RelationshipMapping(RelationshipType.FollowedBy, includeBasedOnRelationship = false)), statsReceiver) @Singleton class ExcludeNonFollowingSgsPredicate @Inject() ( socialGraph: SocialGraph, statsReceiver: StatsReceiver) extends SgsRelationshipsByUserIdPredicate( socialGraph, Seq(RelationshipMapping(RelationshipType.Following, includeBasedOnRelationship = false)), statsReceiver) @Singleton class ExcludeFollowingSgsPredicate @Inject() ( socialGraph: SocialGraph, statsReceiver: StatsReceiver) extends SgsRelationshipsByUserIdPredicate( socialGraph, Seq(RelationshipMapping(RelationshipType.Following, includeBasedOnRelationship = true)), statsReceiver) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsPredicate.scala ================================================ package com.twitter.follow_recommendations.common.predicates.sgs import com.google.common.annotations.VisibleForTesting import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.Predicate import com.twitter.follow_recommendations.common.base.PredicateResult import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasProfileId import com.twitter.follow_recommendations.common.models.FilterReason.FailOpen import com.twitter.follow_recommendations.common.models.FilterReason.InvalidRelationshipTypes import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.socialgraph.thriftscala.ExistsRequest import com.twitter.socialgraph.thriftscala.ExistsResult import com.twitter.socialgraph.thriftscala.LookupContext import com.twitter.socialgraph.thriftscala.Relationship import com.twitter.socialgraph.thriftscala.RelationshipType import com.twitter.stitch.Stitch import com.twitter.stitch.socialgraph.SocialGraph import com.twitter.timelines.configapi.HasParams import com.twitter.util.TimeoutException import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton case class RelationshipMapping( relationshipType: RelationshipType, includeBasedOnRelationship: Boolean) class SgsRelationshipsPredicate( socialGraph: SocialGraph, relationshipMappings: Seq[RelationshipMapping], statsReceiver: StatsReceiver = NullStatsReceiver) extends Predicate[(HasClientContext with HasParams, CandidateUser)] with Logging { private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getSimpleName) override def apply( pair: (HasClientContext with HasParams, CandidateUser) ): Stitch[PredicateResult] = { val (target, candidate) = pair val timeout = target.params(SgsPredicateParams.SgsRelationshipsPredicateTimeout) SgsRelationshipsPredicate .extractUserId(target) .map { id => val relationships = relationshipMappings.map { relationshipMapping: RelationshipMapping => Relationship( relationshipMapping.relationshipType, relationshipMapping.includeBasedOnRelationship) } val existsRequest = ExistsRequest( id, candidate.id, relationships = relationships, context = SgsRelationshipsPredicate.UnionLookupContext ) socialGraph .exists(existsRequest).map { existsResult: ExistsResult => if (existsResult.exists) { PredicateResult.Invalid(Set(InvalidRelationshipTypes(relationshipMappings .map { relationshipMapping: RelationshipMapping => relationshipMapping.relationshipType }.mkString(", ")))) } else { PredicateResult.Valid } } .within(timeout)(com.twitter.finagle.util.DefaultTimer) } // if no user id is present, return true by default .getOrElse(Stitch.value(PredicateResult.Valid)) .rescue { case e: TimeoutException => stats.counter("timeout").incr() Stitch(PredicateResult.Invalid(Set(FailOpen))) case e: Exception => stats.counter(e.getClass.getSimpleName).incr() Stitch(PredicateResult.Invalid(Set(FailOpen))) } } } object SgsRelationshipsPredicate { // OR Operation @VisibleForTesting private[follow_recommendations] val UnionLookupContext = Some( LookupContext(performUnion = Some(true))) private def extractUserId(target: HasClientContext with HasParams): Option[Long] = target match { case profRequest: HasProfileId => Some(profRequest.profileId) case userRequest: HasClientContext with HasParams => userRequest.getOptionalUserId case _ => None } } @Singleton class InvalidTargetCandidateRelationshipTypesPredicate @Inject() ( socialGraph: SocialGraph) extends SgsRelationshipsPredicate( socialGraph, InvalidRelationshipTypesPredicate.InvalidRelationshipTypes) {} @Singleton class NoteworthyAccountsSgsPredicate @Inject() ( socialGraph: SocialGraph) extends SgsRelationshipsPredicate( socialGraph, InvalidRelationshipTypesPredicate.NoteworthyAccountsInvalidRelationshipTypes) object InvalidRelationshipTypesPredicate { val InvalidRelationshipTypesExcludeFollowing: Seq[RelationshipMapping] = Seq( RelationshipMapping(RelationshipType.HideRecommendations, true), RelationshipMapping(RelationshipType.Blocking, true), RelationshipMapping(RelationshipType.BlockedBy, true), RelationshipMapping(RelationshipType.Muting, true), RelationshipMapping(RelationshipType.MutedBy, true), RelationshipMapping(RelationshipType.ReportedAsSpam, true), RelationshipMapping(RelationshipType.ReportedAsSpamBy, true), RelationshipMapping(RelationshipType.ReportedAsAbuse, true), RelationshipMapping(RelationshipType.ReportedAsAbuseBy, true) ) val InvalidRelationshipTypes: Seq[RelationshipMapping] = Seq( RelationshipMapping(RelationshipType.FollowRequestOutgoing, true), RelationshipMapping(RelationshipType.Following, true), RelationshipMapping( RelationshipType.UsedToFollow, true ) // this data is accessible for 90 days. ) ++ InvalidRelationshipTypesExcludeFollowing val NoteworthyAccountsInvalidRelationshipTypes: Seq[RelationshipMapping] = Seq( RelationshipMapping(RelationshipType.Blocking, true), RelationshipMapping(RelationshipType.BlockedBy, true), RelationshipMapping(RelationshipType.Muting, true), RelationshipMapping(RelationshipType.MutedBy, true), RelationshipMapping(RelationshipType.ReportedAsSpam, true), RelationshipMapping(RelationshipType.ReportedAsSpamBy, true), RelationshipMapping(RelationshipType.ReportedAsAbuse, true), RelationshipMapping(RelationshipType.ReportedAsAbuseBy, true) ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", "strato/config/columns/onboarding:onboarding-strato-client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicate.scala ================================================ package com.twitter.follow_recommendations.common.predicates.user_activity import com.twitter.core_workflows.user_model.thriftscala.UserState import com.twitter.decider.Decider import com.twitter.decider.RandomRecipient import com.twitter.finagle.Memcached.Client import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.Predicate import com.twitter.follow_recommendations.common.base.PredicateResult import com.twitter.follow_recommendations.common.base.StatsUtil import com.twitter.follow_recommendations.common.clients.cache.MemcacheClient import com.twitter.follow_recommendations.common.clients.cache.ThriftEnumOptionBijection import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FilterReason import com.twitter.follow_recommendations.configapi.deciders.DeciderKey import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.onboarding.UserRecommendabilityWithLongKeysOnUserClientColumn import com.twitter.timelines.configapi.HasParams import javax.inject.Inject import javax.inject.Singleton abstract case class UserStateActivityPredicate( userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn, validCandidateStates: Set[UserState], client: Client, statsReceiver: StatsReceiver, decider: Decider = Decider.False) extends Predicate[(HasParams with HasClientContext, CandidateUser)] { private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getSimpleName) // client to memcache cluster val bijection = new ThriftEnumOptionBijection[UserState](UserState.apply) val memcacheClient = MemcacheClient[Option[UserState]]( client = client, dest = "/s/cache/follow_recos_service:twemcaches", valueBijection = bijection, ttl = UserActivityPredicateParams.CacheTTL, statsReceiver = stats.scope("twemcache") ) override def apply( targetAndCandidate: (HasParams with HasClientContext, CandidateUser) ): Stitch[PredicateResult] = { val userRecommendabilityFetcher = userRecommendabilityClient.fetcher val (_, candidate) = targetAndCandidate val deciderKey: String = DeciderKey.EnableExperimentalCaching.toString val enableDistributedCaching: Boolean = decider.isAvailable(deciderKey, Some(RandomRecipient)) val userStateStitch: Stitch[Option[UserState]] = enableDistributedCaching match { case true => { memcacheClient.readThrough( // add a key prefix to address cache key collisions key = "UserActivityPredicate" + candidate.id.toString, underlyingCall = () => queryUserRecommendable(candidate.id) ) } case false => queryUserRecommendable(candidate.id) } val resultStitch: Stitch[PredicateResult] = userStateStitch.map { userStateOpt => userStateOpt match { case Some(userState) => { if (validCandidateStates.contains(userState)) { PredicateResult.Valid } else { PredicateResult.Invalid(Set(FilterReason.MinStateNotMet)) } } case None => { PredicateResult.Invalid(Set(FilterReason.MissingRecommendabilityData)) } } } StatsUtil.profileStitch(resultStitch, stats.scope("apply")) .rescue { case e: Exception => stats.scope("rescued").counter(e.getClass.getSimpleName).incr() Stitch(PredicateResult.Invalid(Set(FilterReason.FailOpen))) } } def queryUserRecommendable( userId: Long ): Stitch[Option[UserState]] = { val userRecommendabilityFetcher = userRecommendabilityClient.fetcher userRecommendabilityFetcher.fetch(userId).map { userCandidate => userCandidate.v.flatMap(_.userState) } } } @Singleton class MinStateUserActivityPredicate @Inject() ( userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn, client: Client, statsReceiver: StatsReceiver) extends UserStateActivityPredicate( userRecommendabilityClient, Set( UserState.Light, UserState.HeavyNonTweeter, UserState.MediumNonTweeter, UserState.HeavyTweeter, UserState.MediumTweeter ), client, statsReceiver ) @Singleton class AllTweeterUserActivityPredicate @Inject() ( userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn, client: Client, statsReceiver: StatsReceiver) extends UserStateActivityPredicate( userRecommendabilityClient, Set( UserState.HeavyTweeter, UserState.MediumTweeter ), client, statsReceiver ) @Singleton class HeavyTweeterUserActivityPredicate @Inject() ( userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn, client: Client, statsReceiver: StatsReceiver) extends UserStateActivityPredicate( userRecommendabilityClient, Set( UserState.HeavyTweeter ), client, statsReceiver ) @Singleton class NonNearZeroUserActivityPredicate @Inject() ( userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn, client: Client, statsReceiver: StatsReceiver) extends UserStateActivityPredicate( userRecommendabilityClient, Set( UserState.New, UserState.VeryLight, UserState.Light, UserState.MediumNonTweeter, UserState.MediumTweeter, UserState.HeavyNonTweeter, UserState.HeavyTweeter ), client, statsReceiver ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicateParams.scala ================================================ package com.twitter.follow_recommendations.common.predicates.user_activity import com.twitter.timelines.configapi.FSParam import com.twitter.util.Duration object UserActivityPredicateParams { case object HeavyTweeterEnabled extends FSParam[Boolean]("user_activity_predicate_heavy_tweeter_enabled", false) val CacheTTL: Duration = Duration.fromHours(6) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/AdhocScoreModificationType.scala ================================================ package com.twitter.follow_recommendations.common.rankers.common /** * To manage the extent of adhoc score modifications, we set a hard limit that from each of the * types below *ONLY ONE* adhoc scorer can be applied to candidates' scores. More details about the * usage is available in [[AdhocRanker]] */ object AdhocScoreModificationType extends Enumeration { type AdhocScoreModificationType = Value // This type of scorer increases the score of a subset of candidates through various policies. val BoostingScorer: AdhocScoreModificationType = Value("boosting") // This type of scorer shuffles candidates randomly according to some distribution. val WeightedRandomSamplingScorer: AdhocScoreModificationType = Value("weighted_random_sampling") // This is added solely for testing purposes and should not be used in production. val InvalidAdhocScorer: AdhocScoreModificationType = Value("invalid_adhoc_scorer") } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/slf4j:slf4j-api", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/DedupCandidates.scala ================================================ package com.twitter.follow_recommendations.common.rankers.common import com.twitter.product_mixer.core.model.common.UniversalNoun import scala.collection.mutable object DedupCandidates { def apply[C <: UniversalNoun[Long]](input: Seq[C]): Seq[C] = { val seen = mutable.HashSet[Long]() input.filter { candidate => seen.add(candidate.id) } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/RankerId.scala ================================================ package com.twitter.follow_recommendations.common.rankers.common object RankerId extends Enumeration { type RankerId = Value val RandomRanker: RankerId = Value("random") // The production PostNUX ML warm-start auto-retraining model ranker val PostNuxProdRanker: RankerId = Value("postnux_prod") val None: RankerId = Value("none") // Sampling from the Placket-Luce distribution. Applied after ranker step. Its ranker id is mainly used for logging. val PlacketLuceSamplingTransformer: RankerId = Value("placket_luce_sampling_transformer") def getRankerByName(name: String): Option[RankerId] = RankerId.values.toSeq.find(_.equals(Value(name))) } /** * ML model based heavy ranker ids. */ object ModelBasedHeavyRankerId { import RankerId._ val HeavyRankerIds: Set[String] = Set( PostNuxProdRanker.toString, ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRanker.scala ================================================ package com.twitter.follow_recommendations.common.rankers.fatigue_ranker import com.twitter.finagle.stats.Counter import com.twitter.finagle.stats.Stat import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.Ranker import com.twitter.follow_recommendations.common.base.StatsUtil import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.common.models.HasWtfImpressions import com.twitter.follow_recommendations.common.models.WtfImpression import com.twitter.follow_recommendations.common.rankers.common.RankerId.RankerId import com.twitter.follow_recommendations.common.rankers.utils.Utils import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.servo.util.MemoizingStatsReceiver import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.util.Time /** * Ranks candidates based on the given weights for each algorithm while preserving the ranks inside each algorithm. * Reorders the ranked list based on recent impressions from recentImpressionRepo * * Note that the penalty is added to the rank of each candidate. To make producer-side experiments * with multiple rankers possible, we modify the scores for each candidate and ranker as: * NewScore(C, R) = -(Rank(C, R) + Impression(C, U) x FatigueFactor), * where C is a candidate, R a ranker and U the target user. * Note also that fatigue penalty is independent of any of the rankers. */ class ImpressionBasedFatigueRanker[ Target <: HasClientContext with HasDisplayLocation with HasParams with HasWtfImpressions ]( fatigueFactor: Int, statsReceiver: StatsReceiver) extends Ranker[Target, CandidateUser] { val name: String = this.getClass.getSimpleName val stats = statsReceiver.scope("impression_based_fatigue_ranker") val droppedStats: MemoizingStatsReceiver = new MemoizingStatsReceiver(stats.scope("hard_drops")) val impressionStats: StatsReceiver = stats.scope("wtf_impressions") val noImpressionCounter: Counter = impressionStats.counter("no_impressions") val oldestImpressionStat: Stat = impressionStats.stat("oldest_sec") override def rank(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] = { StatsUtil.profileStitch( Stitch.value(rankCandidates(target, candidates)), stats.scope("rank") ) } private def trackTimeSinceOldestImpression(impressions: Seq[WtfImpression]): Unit = { val timeSinceOldest = Time.now - impressions.map(_.latestTime).min oldestImpressionStat.add(timeSinceOldest.inSeconds) } private def rankCandidates( target: Target, candidates: Seq[CandidateUser] ): Seq[CandidateUser] = { target.wtfImpressions .map { wtfImpressions => if (wtfImpressions.isEmpty) { noImpressionCounter.incr() candidates } else { val rankerIds = candidates.flatMap(_.scores.map(_.scores.flatMap(_.rankerId))).flatten.sorted.distinct /** * In below we create a Map from each CandidateUser's ID to a Map from each Ranker that * the user has a score for, and candidate's corresponding rank when candidates are sorted * by that Ranker (Only candidates who have this Ranker are considered for ranking). */ val candidateRanks: Map[Long, Map[RankerId, Int]] = rankerIds .flatMap { rankerId => // Candidates with no scores from this Ranker is first removed to calculate ranks. val relatedCandidates = candidates.filter(_.scores.exists(_.scores.exists(_.rankerId.contains(rankerId)))) relatedCandidates .sortBy(-_.scores .flatMap(_.scores.find(_.rankerId.contains(rankerId)).map(_.value)).getOrElse( 0.0)).zipWithIndex.map { case (candidate, rank) => (candidate.id, rankerId, rank) } }.groupBy(_._1).map { case (candidate, ranksForAllRankers) => ( candidate, ranksForAllRankers.map { case (_, rankerId, rank) => (rankerId, rank) }.toMap) } val idFatigueCountMap = wtfImpressions.groupBy(_.candidateId).mapValues(_.map(_.counts).sum) trackTimeSinceOldestImpression(wtfImpressions) val rankedCandidates: Seq[CandidateUser] = candidates .map { candidate => val candidateImpressions = idFatigueCountMap.getOrElse(candidate.id, 0) val fatiguedScores = candidate.scores.map { ss => ss.copy(scores = ss.scores.map { s => s.rankerId match { // We set the new score as -rank after fatigue penalty is applied. case Some(rankerId) => // If the candidate's ID is not in the candidate->ranks map, or there is no // rank for this specific ranker and this candidate, we use maximum possible // rank instead. Note that this indicates that there is a problem. s.copy(value = -(candidateRanks .getOrElse(candidate.id, Map()).getOrElse(rankerId, candidates.length) + candidateImpressions * fatigueFactor)) // In case a score exists without a RankerId, we pass on the score as is. case None => s } }) } candidate.copy(scores = fatiguedScores) }.zipWithIndex.map { // We re-rank candidates with their input ordering (which is done by the request-level // ranker) and fatigue penalty. case (candidate, inputRank) => val candidateImpressions = idFatigueCountMap.getOrElse(candidate.id, 0) (candidate, inputRank + candidateImpressions * fatigueFactor) }.sortBy(_._2).map(_._1) // Only populate ranking info when WTF impression info present val scribeRankingInfo: Boolean = target.params(ImpressionBasedFatigueRankerParams.ScribeRankingInfoInFatigueRanker) if (scribeRankingInfo) Utils.addRankingInfo(rankedCandidates, name) else rankedCandidates } }.getOrElse(candidates) // no reranking/filtering when wtf impressions not present } } object ImpressionBasedFatigueRanker { val DefaultFatigueFactor = 5 def build[ Target <: HasClientContext with HasDisplayLocation with HasParams with HasWtfImpressions ]( baseStatsReceiver: StatsReceiver, fatigueFactor: Int = DefaultFatigueFactor ): ImpressionBasedFatigueRanker[Target] = new ImpressionBasedFatigueRanker(fatigueFactor, baseStatsReceiver) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.rankers.fatigue_ranker import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @Singleton class ImpressionBasedFatigueRankerFSConfig @Inject() extends FeatureSwitchConfig { override val booleanFSParams: Seq[FSParam[Boolean]] = Seq(ImpressionBasedFatigueRankerParams.ScribeRankingInfoInFatigueRanker) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerParams.scala ================================================ package com.twitter.follow_recommendations.common.rankers.fatigue_ranker import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.Param object ImpressionBasedFatigueRankerParams { // Whether to enable hard dropping of impressed candidates object DropImpressedCandidateEnabled extends Param[Boolean](false) // At what # of impressions to hard drop candidates. object DropCandidateImpressionThreshold extends Param[Int](default = 10) // Whether to scribe candidate ranking/scoring info per ranking stage object ScribeRankingInfoInFatigueRanker extends FSParam[Boolean]("fatigue_ranker_scribe_ranking_info", true) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRanker.scala ================================================ package com.twitter.follow_recommendations.common.rankers.first_n_ranker import com.google.inject.Inject import com.google.inject.Singleton import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.Ranker import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasQualityFactor import com.twitter.follow_recommendations.common.rankers.utils.Utils import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams /** * This class is meant to filter candidates between stages of our ranker by taking the first N * candidates, merging any candidate source information for candidates with multiple entries. * To allow us to chain this truncation operation any number of times sequentially within the main * ranking builder, we abstract the truncation as a separate Ranker */ @Singleton class FirstNRanker[Target <: HasClientContext with HasParams with HasQualityFactor] @Inject() ( stats: StatsReceiver) extends Ranker[Target, CandidateUser] { val name: String = this.getClass.getSimpleName private val baseStats = stats.scope("first_n_ranker") val scaledDownByQualityFactorCounter = baseStats.counter("scaled_down_by_quality_factor") private val mergeStat = baseStats.scope("merged_candidates") private val mergeStat2 = mergeStat.counter("2") private val mergeStat3 = mergeStat.counter("3") private val mergeStat4 = mergeStat.counter("4+") private val candidateSizeStats = baseStats.scope("candidate_size") private case class CandidateSourceScore( candidateId: Long, sourceId: CandidateSourceIdentifier, score: Option[Double]) /** * Adds the rank of each candidate based on the primary candidate source's score. * In the event where the provided ordering of candidates do not align with the score, * we will respect the score, since the ordering might have been mixed up due to other previous * steps like the shuffleFn in the `WeightedCandidateSourceRanker`. * @param candidates ordered list of candidates * @return same ordered list of candidates, but with the rank information appended */ def addRank(candidates: Seq[CandidateUser]): Seq[CandidateUser] = { val candidateSourceRanks = for { (sourceIdOpt, sourceCandidates) <- candidates.groupBy(_.getPrimaryCandidateSource) (candidate, rank) <- sourceCandidates.sortBy(-_.score.getOrElse(0.0)).zipWithIndex } yield { (candidate, sourceIdOpt) -> rank } candidates.map { c => c.getPrimaryCandidateSource .map { sourceId => val sourceRank = candidateSourceRanks((c, c.getPrimaryCandidateSource)) c.addCandidateSourceRanksMap(Map(sourceId -> sourceRank)) }.getOrElse(c) } } override def rank(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] = { val scaleDownFactor = Math.max( target.qualityFactor.getOrElse(1.0d), target.params(FirstNRankerParams.MinNumCandidatesScoredScaleDownFactor) ) if (scaleDownFactor < 1.0d) scaledDownByQualityFactorCounter.incr() val n = (target.params(FirstNRankerParams.CandidatesToRank) * scaleDownFactor).toInt val scribeRankingInfo: Boolean = target.params(FirstNRankerParams.ScribeRankingInfoInFirstNRanker) candidateSizeStats.counter(s"n$n").incr() val candidatesWithRank = addRank(candidates) if (target.params(FirstNRankerParams.GroupDuplicateCandidates)) { val groupedCandidates: Map[Long, Seq[CandidateUser]] = candidatesWithRank.groupBy(_.id) val topN = candidates .map { c => merge(groupedCandidates(c.id)) }.distinct.take(n) Stitch.value(if (scribeRankingInfo) Utils.addRankingInfo(topN, name) else topN) } else { Stitch.value( if (scribeRankingInfo) Utils.addRankingInfo(candidatesWithRank, name).take(n) else candidatesWithRank.take(n)) } // for efficiency, if don't need to deduplicate } /** * we use the primary candidate source of the first entry, and aggregate all of the other entries' * candidate source scores into the first entry's candidateSourceScores * @param candidates list of candidates with the same id * @return a single merged candidate */ private[first_n_ranker] def merge(candidates: Seq[CandidateUser]): CandidateUser = { if (candidates.size == 1) { candidates.head } else { candidates.size match { case 2 => mergeStat2.incr() case 3 => mergeStat3.incr() case i if i >= 4 => mergeStat4.incr() case _ => } val allSources = candidates.flatMap(_.getCandidateSources).toMap val allRanks = candidates.flatMap(_.getCandidateRanks).toMap candidates.head.addCandidateSourceScoresMap(allSources).addCandidateSourceRanksMap(allRanks) } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.rankers.first_n_ranker import javax.inject.Inject import javax.inject.Singleton import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam @Singleton class FirstNRankerFSConfig @Inject() extends FeatureSwitchConfig { override val booleanFSParams: Seq[FSParam[Boolean]] = Seq(FirstNRankerParams.ScribeRankingInfoInFirstNRanker) override val intFSParams: Seq[FSBoundedParam[Int]] = Seq( FirstNRankerParams.CandidatesToRank ) override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( FirstNRankerParams.MinNumCandidatesScoredScaleDownFactor ) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFeatureSwitchKeys.scala ================================================ package com.twitter.follow_recommendations.common.rankers.first_n_ranker object FirstNRankerFeatureSwitchKeys { val CandidatePoolSize = "first_n_ranker_candidate_pool_size" val ScribeRankingInfo = "first_n_ranker_scribe_ranking_info" val MinNumCandidatesScoredScaleDownFactor = "first_n_ranker_min_scale_down_factor" } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerParams.scala ================================================ package com.twitter.follow_recommendations.common.rankers.first_n_ranker import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.Param object FirstNRankerParams { case object CandidatesToRank extends FSBoundedParam[Int]( FirstNRankerFeatureSwitchKeys.CandidatePoolSize, default = 100, min = 50, max = 600) case object GroupDuplicateCandidates extends Param[Boolean](true) case object ScribeRankingInfoInFirstNRanker extends FSParam[Boolean](FirstNRankerFeatureSwitchKeys.ScribeRankingInfo, true) // the minimum of candidates to score in each request. object MinNumCandidatesScoredScaleDownFactor extends FSBoundedParam[Double]( name = FirstNRankerFeatureSwitchKeys.MinNumCandidatesScoredScaleDownFactor, default = 0.3, min = 0.1, max = 1.0) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRanker.scala ================================================ package com.twitter.follow_recommendations.common.rankers.interleave_ranker import com.google.common.annotations.VisibleForTesting import com.google.inject.Inject import com.google.inject.Singleton import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.Ranker import com.twitter.follow_recommendations.common.base.StatsUtil import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.rankers.common.RankerId import com.twitter.follow_recommendations.common.rankers.utils.Utils import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams @Singleton class InterleaveRanker[Target <: HasParams] @Inject() ( statsReceiver: StatsReceiver) extends Ranker[Target, CandidateUser] { val name: String = this.getClass.getSimpleName private val stats = statsReceiver.scope("interleave_ranker") private val inputStats = stats.scope("input") private val interleavingStats = stats.scope("interleave") override def rank( target: Target, candidates: Seq[CandidateUser] ): Stitch[Seq[CandidateUser]] = { StatsUtil.profileStitch( Stitch.value(rankCandidates(target, candidates)), stats.scope("rank") ) } private def rankCandidates( target: Target, candidates: Seq[CandidateUser] ): Seq[CandidateUser] = { /** * By this stage, all valid candidates should have: * 1. Their Scores field populated. * 2. Their selectedRankerId set. * 3. Have a score associated to their selectedRankerId. * If there is any candidate that doesn't meet the conditions above, there is a problem in one * of the previous rankers. Since no new scoring is done in this ranker, we simply remove them. */ val validCandidates = candidates.filter { c => c.scores.isDefined && c.scores.exists(_.selectedRankerId.isDefined) && getCandidateScoreByRankerId(c, c.scores.flatMap(_.selectedRankerId)).isDefined } // To monitor the percentage of valid candidates, as defined above, we track the following: inputStats.counter("candidates_with_no_scores").incr(candidates.count(_.scores.isEmpty)) inputStats .counter("candidates_with_no_selected_ranker").incr(candidates.count { c => c.scores.isEmpty || c.scores.exists(_.selectedRankerId.isEmpty) }) inputStats .counter("candidates_with_no_score_for_selected_ranker").incr(candidates.count { c => c.scores.isEmpty || c.scores.exists(_.selectedRankerId.isEmpty) || getCandidateScoreByRankerId(c, c.scores.flatMap(_.selectedRankerId)).isEmpty }) inputStats.counter("total_num_candidates").incr(candidates.length) inputStats.counter("total_valid_candidates").incr(validCandidates.length) // We only count rankerIds from those candidates who are valid to exclude those candidates with // a valid selectedRankerId that don't have an associated score for it. val rankerIds = validCandidates.flatMap(_.scores.flatMap(_.selectedRankerId)).sorted.distinct rankerIds.foreach { rankerId => inputStats .counter(s"valid_scores_for_${rankerId.toString}").incr( candidates.count(getCandidateScoreByRankerId(_, Some(rankerId)).isDefined)) inputStats.counter(s"total_candidates_for_${rankerId.toString}").incr(candidates.length) } inputStats.counter(s"num_ranker_ids=${rankerIds.length}").incr() val scribeRankingInfo: Boolean = target.params(InterleaveRankerParams.ScribeRankingInfoInInterleaveRanker) if (rankerIds.length <= 1) // In the case of "Number of RankerIds = 0", we pass on the candidates even though there is // a problem in a previous ranker that provided the scores. if (scribeRankingInfo) Utils.addRankingInfo(candidates, name) else candidates else if (scribeRankingInfo) Utils.addRankingInfo(interleaveCandidates(validCandidates, rankerIds), name) else interleaveCandidates(validCandidates, rankerIds) } @VisibleForTesting private[interleave_ranker] def interleaveCandidates( candidates: Seq[CandidateUser], rankerIds: Seq[RankerId.RankerId] ): Seq[CandidateUser] = { val candidatesWithRank = rankerIds .flatMap { ranker => candidates // We first sort all candidates using this ranker. .sortBy(-getCandidateScoreByRankerId(_, Some(ranker)).getOrElse(Double.MinValue)) .zipWithIndex.filter( // but only hold those candidates whose selected ranker is this ranker. // These ranks will be forced in the final ordering. _._1.scores.flatMap(_.selectedRankerId).contains(ranker)) } // Only candidates who have isInProducerScoringExperiment set to true will have their position enforced. We // separate candidates into two groups: (1) Production and (2) Experiment. val (expCandidates, prodCandidates) = candidatesWithRank.partition(_._1.scores.exists(_.isInProducerScoringExperiment)) // We resolve (potential) conflicts between the enforced ranks of experimental models. val expCandidatesFinalPos = resolveConflicts(expCandidates) // Retrieve non-occupied positions and assign them to candidates who use production ranker. val occupiedPos = expCandidatesFinalPos.map(_._2).toSet val prodCandidatesFinalPos = prodCandidates .map(_._1).zip( candidates.indices.filterNot(occupiedPos.contains).sorted.take(prodCandidates.length)) // Merge the two groups and sort them by their corresponding positions. val finalCandidates = (prodCandidatesFinalPos ++ expCandidatesFinalPos).sortBy(_._2).map(_._1) // We count the presence of each ranker in the top-3 final positions. finalCandidates.zip(0 until 3).foreach { case (c, r) => // We only do so for candidates that are in a producer-side experiment. if (c.scores.exists(_.isInProducerScoringExperiment)) c.scores.flatMap(_.selectedRankerId).map(_.toString).foreach { rankerName => interleavingStats .counter(s"num_final_position_${r}_$rankerName") .incr() } } finalCandidates } @VisibleForTesting private[interleave_ranker] def resolveConflicts( candidatesWithRank: Seq[(CandidateUser, Int)] ): Seq[(CandidateUser, Int)] = { // The following two metrics will allow us to calculate the rate of conflicts occurring. // Example: If overall there are 10 producers in different bucketing experiments, and 3 of them // are assigned to the same position. The rate would be 3/10, 30%. val numCandidatesWithConflicts = interleavingStats.counter("candidates_with_conflict") val numCandidatesNoConflicts = interleavingStats.counter("candidates_without_conflict") val candidatesGroupedByRank = candidatesWithRank.groupBy(_._2).toSeq.sortBy(_._1).map { case (rank, candidatesWithRank) => (rank, candidatesWithRank.map(_._1)) } candidatesGroupedByRank.foldLeft(Seq[(CandidateUser, Int)]()) { (upToHere, nextGroup) => val (rank, candidates) = nextGroup if (candidates.length > 1) numCandidatesWithConflicts.incr(candidates.length) else numCandidatesNoConflicts.incr() // We use the position after the last-assigned candidate as a starting point, or 0 otherwise. // If candidates' position is after this "starting point", we enforce that position instead. val minAvailableIndex = scala.math.max(upToHere.lastOption.map(_._2).getOrElse(-1) + 1, rank) val enforcedPos = (minAvailableIndex until minAvailableIndex + candidates.length).toList val shuffledEnforcedPos = if (candidates.length > 1) scala.util.Random.shuffle(enforcedPos) else enforcedPos if (shuffledEnforcedPos.length > 1) { candidates.zip(shuffledEnforcedPos).sortBy(_._2).map(_._1).zipWithIndex.foreach { case (c, r) => c.scores.flatMap(_.selectedRankerId).map(_.toString).foreach { rankerName => // For each ranker, we count the total number of times it has been in a conflict. interleavingStats .counter(s"num_${shuffledEnforcedPos.length}-way_conflicts_$rankerName") .incr() // We also count the positions each of the rankers have fallen randomly into. In any // experiment this should converge to uniform distribution given enough occurrences. // Note that the position here is relative to the other candidates in the conflict and // not the overall position of each candidate. interleavingStats .counter( s"num_position_${r}_after_${shuffledEnforcedPos.length}-way_conflict_$rankerName") .incr() } } } upToHere ++ candidates.zip(shuffledEnforcedPos).sortBy(_._2) } } @VisibleForTesting private[interleave_ranker] def getCandidateScoreByRankerId( candidate: CandidateUser, rankerIdOpt: Option[RankerId.RankerId] ): Option[Double] = { rankerIdOpt match { case None => None case Some(rankerId) => candidate.scores.flatMap { _.scores.find(_.rankerId.contains(rankerId)).map(_.value) } } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.rankers.interleave_ranker import javax.inject.Inject import javax.inject.Singleton import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSParam @Singleton class InterleaveRankerFSConfig @Inject() extends FeatureSwitchConfig { override val booleanFSParams: Seq[FSParam[Boolean]] = Seq(InterleaveRankerParams.ScribeRankingInfoInInterleaveRanker) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerParams.scala ================================================ package com.twitter.follow_recommendations.common.rankers.interleave_ranker import com.twitter.timelines.configapi.FSParam object InterleaveRankerParams { case object ScribeRankingInfoInInterleaveRanker extends FSParam[Boolean]("interleave_ranker_scribe_ranking_info", true) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", "src/java/com/twitter/ml/api:api-base", "util/util-slf4j-api/src/main/scala", ], ) # This is to import only the params from MlRanker, for instance to get request-level heavy ranker. scala_library( name = "ml_ranker_params", sources = [ "MlRankerParams.scala", ], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", "timelines/src/main/scala/com/twitter/timelines/config/configapi", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/HydrateFeaturesTransform.scala ================================================ package com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking import com.google.inject.Inject import com.google.inject.Singleton import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.GatedTransform import com.twitter.follow_recommendations.common.base.StatsUtil.profileStitchMapResults import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature import com.twitter.follow_recommendations.common.feature_hydration.sources.UserScoringFeatureSource import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDebugOptions import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.common.models.HasSimilarToContext import com.twitter.follow_recommendations.common.models.RichDataRecord import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.util.logging.Logging /** * Hydrate features given target and candidates lists. * This is a required step before MlRanker. * If a feature is not hydrated before MlRanker is triggered, a runtime exception will be thrown */ @Singleton class HydrateFeaturesTransform[ Target <: HasClientContext with HasParams with HasDebugOptions with HasPreFetchedFeature with HasSimilarToContext with HasDisplayLocation] @Inject() ( userScoringFeatureSource: UserScoringFeatureSource, stats: StatsReceiver) extends GatedTransform[Target, CandidateUser] with Logging { private val hydrateFeaturesStats = stats.scope("hydrate_features") def transform(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] = { // get features val featureMapStitch: Stitch[Map[CandidateUser, DataRecord]] = profileStitchMapResults( userScoringFeatureSource.hydrateFeatures(target, candidates), hydrateFeaturesStats) featureMapStitch.map { featureMap => candidates .map { candidate => val dataRecord = featureMap(candidate) // add debugRecord only when the request parameter is set val debugDataRecord = if (target.debugOptions.exists(_.fetchDebugInfo)) { Some(candidate.toDebugDataRecord(dataRecord, userScoringFeatureSource.featureContext)) } else None candidate.copy( dataRecord = Some(RichDataRecord(Some(dataRecord), debugDataRecord)) ) } } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRanker.scala ================================================ package com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking import com.google.common.annotations.VisibleForTesting import com.google.inject.Inject import com.google.inject.Singleton import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.Ranker import com.twitter.follow_recommendations.common.base.StatsUtil import com.twitter.follow_recommendations.common.base.StatsUtil.profileSeqResults import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.common.models.HasDebugOptions import com.twitter.follow_recommendations.common.models.Scores import com.twitter.follow_recommendations.common.rankers.common.RankerId import com.twitter.follow_recommendations.common.rankers.common.RankerId.RankerId import com.twitter.follow_recommendations.common.rankers.utils.Utils import com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring.AdhocScorer import com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring.Scorer import com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring.ScorerFactory import com.twitter.follow_recommendations.common.utils.CollectionUtil import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.timelines.configapi.Params import com.twitter.util.logging.Logging /** * This class has a rank function that will perform 4 steps: * - choose which scorer to use for each candidate * - score candidates given their respective features * - add scoring information to the candidate * - sort candidates by their respective scores * The feature source and scorer will depend on the request's params */ @Singleton class MlRanker[ Target <: HasClientContext with HasParams with HasDisplayLocation with HasDebugOptions] @Inject() ( scorerFactory: ScorerFactory, statsReceiver: StatsReceiver) extends Ranker[Target, CandidateUser] with Logging { private val stats: StatsReceiver = statsReceiver.scope("ml_ranker") private val inputStat = stats.scope("1_input") private val selectScorerStat = stats.scope("2_select_scorer") private val scoreStat = stats.scope("3_score") override def rank( target: Target, candidates: Seq[CandidateUser] ): Stitch[Seq[CandidateUser]] = { profileSeqResults(candidates, inputStat) val requestRankerId = target.params(MlRankerParams.RequestScorerIdParam) val rankerIds = chooseRankerByCandidate(candidates, requestRankerId) val scoreStitch = score(candidates, rankerIds, requestRankerId).map { scoredCandidates => { // sort the candidates by score val sortedCandidates = sort(target, scoredCandidates) // add scribe field to candidates (if applicable) and return candidates scribeCandidates(target, sortedCandidates) } } StatsUtil.profileStitch(scoreStitch, stats.scope("rank")) } /** * @param target: The WTF request for a given consumer. * @param candidates A list of candidates considered for recommendation. * @return A map from each candidate to a tuple that includes: * (1) The selected scorer that should be used to rank this candidate * (2) a flag determining whether the candidate is in a producer-side experiment. */ private[ranking] def chooseRankerByCandidate( candidates: Seq[CandidateUser], requestRankerId: RankerId ): Map[CandidateUser, RankerId] = { candidates.map { candidate => val selectedCandidateRankerId = if (candidate.params == Params.Invalid || candidate.params == Params.Empty) { selectScorerStat.counter("candidate_params_empty").incr() requestRankerId } else { val candidateRankerId = candidate.params(MlRankerParams.CandidateScorerIdParam) if (candidateRankerId == RankerId.None) { // This candidate is a not part of any producer-side experiment. selectScorerStat.counter("default_to_request_ranker").incr() requestRankerId } else { // This candidate is in a treatment bucket of a producer-side experiment. selectScorerStat.counter("use_candidate_ranker").incr() candidateRankerId } } selectScorerStat.scope("selected").counter(selectedCandidateRankerId.toString).incr() candidate -> selectedCandidateRankerId }.toMap } @VisibleForTesting private[ranking] def score( candidates: Seq[CandidateUser], rankerIds: Map[CandidateUser, RankerId], requestRankerId: RankerId ): Stitch[Seq[CandidateUser]] = { val features = candidates.map(_.dataRecord.flatMap(_.dataRecord)) require(features.forall(_.nonEmpty), "features are not hydrated for all the candidates") val scorers = scorerFactory.getScorers(rankerIds.values.toSeq.sorted.distinct) // Scorers are split into ML-based and Adhoc (defined as a scorer that does not need to call an // ML prediction service and scores candidates using locally-available data). val (adhocScorers, mlScorers) = scorers.partition { case _: AdhocScorer => true case _ => false } // score candidates val scoresStitch = score(features.map(_.get), mlScorers) val candidatesWithMlScoresStitch = scoresStitch.map { scoresSeq => candidates .zip(scoresSeq).map { // copy datarecord and score into candidate object case (candidate, scores) => val selectedRankerId = rankerIds(candidate) val useRequestRanker = candidate.params == Params.Invalid || candidate.params == Params.Empty || candidate.params(MlRankerParams.CandidateScorerIdParam) == RankerId.None candidate.copy( score = scores.scores.find(_.rankerId.contains(requestRankerId)).map(_.value), scores = if (scores.scores.nonEmpty) { Some( scores.copy( scores = scores.scores, selectedRankerId = Some(selectedRankerId), isInProducerScoringExperiment = !useRequestRanker )) } else None ) } } candidatesWithMlScoresStitch.map { candidates => // The basis for adhoc scores are the "request-level" ML ranker. We add the base score here // while adhoc scorers are applied in [[AdhocRanker]]. addMlBaseScoresForAdhocScorers(candidates, requestRankerId, adhocScorers) } } @VisibleForTesting private[ranking] def addMlBaseScoresForAdhocScorers( candidates: Seq[CandidateUser], requestRankerId: RankerId, adhocScorers: Seq[Scorer] ): Seq[CandidateUser] = { candidates.map { candidate => candidate.scores match { case Some(oldScores) => // 1. We fetch the ML score that is the basis of adhoc scores: val baseMlScoreOpt = Utils.getCandidateScoreByRankerId(candidate, requestRankerId) // 2. For each adhoc scorer, we copy the ML score object, changing only the ID and type. val newScores = adhocScorers flatMap { adhocScorer => baseMlScoreOpt.map( _.copy(rankerId = Some(adhocScorer.id), scoreType = adhocScorer.scoreType)) } // 3. We add the new adhoc score entries to the candidate. candidate.copy(scores = Some(oldScores.copy(scores = oldScores.scores ++ newScores))) case _ => // Since there is no base ML score, there should be no adhoc score modification as well. candidate } } } private[this] def score( dataRecords: Seq[DataRecord], scorers: Seq[Scorer] ): Stitch[Seq[Scores]] = { val scoredResponse = scorers.map { scorer => StatsUtil.profileStitch(scorer.score(dataRecords), scoreStat.scope(scorer.id.toString)) } // If we could score a candidate with too many rankers, it is likely to blow up the whole system. // and fail back to default production model StatsUtil.profileStitch(Stitch.collect(scoredResponse), scoreStat).map { scoresByScorerId => CollectionUtil.transposeLazy(scoresByScorerId).map { scoresPerCandidate => Scores(scoresPerCandidate) } } } // sort candidates using score in descending order private[this] def sort( target: Target, candidates: Seq[CandidateUser] ): Seq[CandidateUser] = { candidates.sortBy(c => -c.score.getOrElse(MlRanker.DefaultScore)) } private[this] def scribeCandidates( target: Target, candidates: Seq[CandidateUser] ): Seq[CandidateUser] = { val scribeRankingInfo: Boolean = target.params(MlRankerParams.ScribeRankingInfoInMlRanker) scribeRankingInfo match { case true => Utils.addRankingInfo(candidates, "MlRanker") case false => candidates } } } object MlRanker { // this is to ensure candidates with absent scores are ranked the last val DefaultScore: Double = Double.MinValue } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking import javax.inject.Inject import javax.inject.Singleton import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSParam @Singleton class MlRankerFSConfig @Inject() extends FeatureSwitchConfig { override val booleanFSParams: Seq[FSParam[Boolean]] = Seq(MlRankerParams.ScribeRankingInfoInMlRanker) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerParams.scala ================================================ package com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking import com.twitter.follow_recommendations.common.rankers.common.RankerId import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSParam /** * When adding Producer side experiments, make sure to register the FS Key in [[ProducerFeatureFilter]] * in [[FeatureSwitchesModule]], otherwise, the FS will not work. */ object MlRankerParams { // which ranker to use by default for the given request case object RequestScorerIdParam extends FSEnumParam[RankerId.type]( name = "post_nux_ml_flow_ml_ranker_id", default = RankerId.PostNuxProdRanker, enum = RankerId ) // which ranker to use for the given candidate case object CandidateScorerIdParam extends FSEnumParam[RankerId.type]( name = "post_nux_ml_flow_candidate_user_scorer_id", default = RankerId.None, enum = RankerId ) case object ScribeRankingInfoInMlRanker extends FSParam[Boolean]("post_nux_ml_flow_scribe_ranking_info_in_ml_ranker", true) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/AdhocScorer.scala ================================================ package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring import com.twitter.follow_recommendations.common.rankers.common.AdhocScoreModificationType.AdhocScoreModificationType import com.twitter.follow_recommendations.common.models.Score import com.twitter.ml.api.DataRecord import com.twitter.stitch.Stitch trait AdhocScorer extends Scorer { /** * NOTE: For instances of [[AdhocScorer]] this function SHOULD NOT be used. * Please use: * [[score(target: HasClientContext with HasParams, candidates: Seq[CandidateUser])]] * instead. */ @Deprecated override def score(records: Seq[DataRecord]): Stitch[Seq[Score]] = throw new UnsupportedOperationException( "For instances of AdhocScorer this operation is not defined. Please use " + "`def score(target: HasClientContext with HasParams, candidates: Seq[CandidateUser])` " + "instead.") /** * This helps us manage the extend of adhoc modification on candidates' score. There is a hard * limit of applying ONLY ONE scorer of each type to a score. */ val scoreModificationType: AdhocScoreModificationType } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking:ml_ranker_params", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/pluck/source/core_workflows/user_model:condensed_user_state-scala", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/DeepbirdScorer.scala ================================================ package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService import com.twitter.cortex.deepbird.thriftjava.ModelSelector import com.twitter.finagle.stats.Stat import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDebugOptions import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.common.models.Score import com.twitter.ml.api.DataRecord import com.twitter.ml.api.Feature import com.twitter.ml.api.RichDataRecord import com.twitter.ml.prediction_service.{BatchPredictionRequest => JBatchPredictionRequest} import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.util.Future import com.twitter.util.TimeoutException import scala.collection.JavaConversions._ import scala.collection.JavaConverters._ /** * Generic trait that implements the scoring given a deepbirdClient * To test out a new model, create a scorer extending this trait, override the modelName and inject the scorer */ trait DeepbirdScorer extends Scorer { def modelName: String def predictionFeature: Feature.Continuous // Set a default batchSize of 100 when making model prediction calls to the Deepbird V2 prediction server def batchSize: Int = 100 def deepbirdClient: DeepbirdPredictionService.ServiceToClient def baseStats: StatsReceiver def modelSelector: ModelSelector = new ModelSelector().setId(modelName) def stats: StatsReceiver = baseStats.scope(this.getClass.getSimpleName).scope(modelName) private def requestCount = stats.counter("requests") private def emptyRequestCount = stats.counter("empty_requests") private def successCount = stats.counter("success") private def failureCount = stats.counter("failures") private def inputRecordsStat = stats.stat("input_records") private def outputRecordsStat = stats.stat("output_records") // Counters for tracking batch-prediction statistics when making DBv2 prediction calls // // numBatchRequests tracks the number of batch prediction requests made to DBv2 prediction servers private def numBatchRequests = stats.counter("batches") // numEmptyBatchRequests tracks the number of batch prediction requests made to DBv2 prediction servers // that had an empty input DataRecord private def numEmptyBatchRequests = stats.counter("empty_batches") // numTimedOutBatchRequests tracks the number of batch prediction requests made to DBv2 prediction servers // that had timed-out private def numTimedOutBatchRequests = stats.counter("timeout_batches") private def batchPredictionLatency = stats.stat("batch_prediction_latency") private def predictionLatency = stats.stat("prediction_latency") private def numEmptyModelPredictions = stats.counter("empty_model_predictions") private def numNonEmptyModelPredictions = stats.counter("non_empty_model_predictions") private val DefaultPredictionScore = 0.0 /** * NOTE: For instances of [[DeepbirdScorer]] this function SHOULD NOT be used. * Please use [[score(records: Seq[DataRecord])]] instead. */ @Deprecated def score( target: HasClientContext with HasParams with HasDisplayLocation with HasDebugOptions, candidates: Seq[CandidateUser] ): Seq[Option[Score]] = throw new UnsupportedOperationException( "For instances of DeepbirdScorer this operation is not defined. Please use " + "`def score(records: Seq[DataRecord]): Stitch[Seq[Score]]` " + "instead.") override def score(records: Seq[DataRecord]): Stitch[Seq[Score]] = { requestCount.incr() if (records.isEmpty) { emptyRequestCount.incr() Stitch.Nil } else { inputRecordsStat.add(records.size) Stitch.callFuture( batchPredict(records, batchSize) .map { recordList => val scores = recordList.map { record => Score( value = record.getOrElse(DefaultPredictionScore), rankerId = Some(id), scoreType = scoreType) } outputRecordsStat.add(scores.size) scores }.onSuccess(_ => successCount.incr()) .onFailure(_ => failureCount.incr())) } } def batchPredict( dataRecords: Seq[DataRecord], batchSize: Int ): Future[Seq[Option[Double]]] = { Stat .timeFuture(predictionLatency) { val batchedDataRecords = dataRecords.grouped(batchSize).toSeq numBatchRequests.incr(batchedDataRecords.size) Future .collect(batchedDataRecords.map(batch => predict(batch))) .map(res => res.reduce(_ ++ _)) } } def predict(dataRecords: Seq[DataRecord]): Future[Seq[Option[Double]]] = { Stat .timeFuture(batchPredictionLatency) { if (dataRecords.isEmpty) { numEmptyBatchRequests.incr() Future.Nil } else { deepbirdClient .batchPredictFromModel(new JBatchPredictionRequest(dataRecords.asJava), modelSelector) .map { response => response.predictions.toSeq.map { prediction => val predictionFeatureOption = Option( new RichDataRecord(prediction).getFeatureValue(predictionFeature) ) predictionFeatureOption match { case Some(predictionValue) => numNonEmptyModelPredictions.incr() Option(predictionValue.toDouble) case None => numEmptyModelPredictions.incr() Option(DefaultPredictionScore) } } } .rescue { case e: TimeoutException => // DBv2 prediction calls that timed out numTimedOutBatchRequests.incr() stats.counter(e.getClass.getSimpleName).incr() Future.value(dataRecords.map(_ => Option(DefaultPredictionScore))) case e: Exception => // other generic DBv2 prediction call failures stats.counter(e.getClass.getSimpleName).incr() Future.value(dataRecords.map(_ => Option(DefaultPredictionScore))) } } } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/PostnuxDeepbirdProdScorer.scala ================================================ package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.follow_recommendations.common.rankers.common.RankerId import com.twitter.ml.api.Feature import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton // This is a standard DeepbirdV2 ML Ranker scoring config that should be extended by all ML scorers // // Only modify this trait when adding new fields to DeepbirdV2 scorers which trait DeepbirdProdScorer extends DeepbirdScorer { override val batchSize = 20 } // Feature.Continuous("prediction") is specific to ClemNet architecture, we can change it to be more informative in the next iteration trait PostNuxV1DeepbirdProdScorer extends DeepbirdProdScorer { override val predictionFeature: Feature.Continuous = new Feature.Continuous("prediction") } // The current, primary PostNUX DeepbirdV2 scorer used in production @Singleton class PostnuxDeepbirdProdScorer @Inject() ( @Named(GuiceNamedConstants.WTF_PROD_DEEPBIRDV2_CLIENT) override val deepbirdClient: DeepbirdPredictionService.ServiceToClient, override val baseStats: StatsReceiver) extends PostNuxV1DeepbirdProdScorer { override val id = RankerId.PostNuxProdRanker override val modelName = "PostNUX14531GafClemNetWarmStart" } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/RandomScorer.scala ================================================ package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.follow_recommendations.common.rankers.common.RankerId import com.twitter.ml.api.DataRecord import com.twitter.ml.api.Feature import com.twitter.util.Future import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton /** * This scorer assigns random values between 0 and 1 to each candidate as scores. */ @Singleton class RandomScorer @Inject() ( @Named(GuiceNamedConstants.WTF_PROD_DEEPBIRDV2_CLIENT) override val deepbirdClient: DeepbirdPredictionService.ServiceToClient, override val baseStats: StatsReceiver) extends DeepbirdScorer { override val id = RankerId.RandomRanker private val rnd = new scala.util.Random(System.currentTimeMillis()) override def predict(dataRecords: Seq[DataRecord]): Future[Seq[Option[Double]]] = { if (dataRecords.isEmpty) { Future.Nil } else { // All candidates are assigned a random value between 0 and 1 as score. Future.value(dataRecords.map(_ => Option(rnd.nextDouble()))) } } override val modelName = "PostNuxRandomRanker" // This is not needed since we are overriding the `predict` function, but we have to override // `predictionFeature` anyway. override val predictionFeature: Feature.Continuous = new Feature.Continuous("prediction.pfollow_pengagement") } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/Scorer.scala ================================================ package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.common.models.HasDebugOptions import com.twitter.follow_recommendations.common.models.Score import com.twitter.follow_recommendations.common.models.ScoreType import com.twitter.follow_recommendations.common.rankers.common.RankerId import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams trait Scorer { // unique id of the scorer def id: RankerId.Value // type of the output scores def scoreType: Option[ScoreType] = None // Scoring when an ML model is used. def score(records: Seq[DataRecord]): Stitch[Seq[Score]] /** * Scoring when a non-ML method is applied. E.g: Boosting, randomized reordering, etc. * This method assumes that candidates' scores are already retrieved from heavy-ranker models and * are available for use. */ def score( target: HasClientContext with HasParams with HasDisplayLocation with HasDebugOptions, candidates: Seq[CandidateUser] ): Seq[Option[Score]] } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/ScorerFactory.scala ================================================ package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.rankers.common.RankerId import com.twitter.follow_recommendations.common.rankers.common.RankerId.RankerId import javax.inject.Inject import javax.inject.Singleton @Singleton class ScorerFactory @Inject() ( postnuxProdScorer: PostnuxDeepbirdProdScorer, randomScorer: RandomScorer, stats: StatsReceiver) { private val scorerFactoryStats = stats.scope("scorer_factory") private val scorerStat = scorerFactoryStats.scope("scorer") def getScorers( rankerIds: Seq[RankerId] ): Seq[Scorer] = { rankerIds.map { scorerId => val scorer: Scorer = getScorerById(scorerId) // count # of times a ranker has been requested scorerStat.counter(scorer.id.toString).incr() scorer } } def getScorerById(scorerId: RankerId): Scorer = scorerId match { case RankerId.PostNuxProdRanker => postnuxProdScorer case RankerId.RandomRanker => randomScorer case _ => scorerStat.counter("invalid_scorer_type").incr() throw new IllegalArgumentException("unknown_scorer_type") } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/Utils.scala ================================================ package com.twitter.follow_recommendations.common.rankers.utils import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.Score import com.twitter.follow_recommendations.common.rankers.common.RankerId.RankerId object Utils { /** * Add the ranking and scoring info for a list of candidates on a given ranking stage. * @param candidates A list of CandidateUser * @param rankingStage Should use `Ranker.name` as the ranking stage. * @return The list of CandidateUser with ranking/scoring info added. */ def addRankingInfo(candidates: Seq[CandidateUser], rankingStage: String): Seq[CandidateUser] = { candidates.zipWithIndex.map { case (candidate, rank) => // 1-based ranking for better readability candidate.addInfoPerRankingStage(rankingStage, candidate.scores, rank + 1) } } def getCandidateScoreByRankerId(candidate: CandidateUser, rankerId: RankerId): Option[Score] = candidate.scores.flatMap { ss => ss.scores.find(_.rankerId.contains(rankerId)) } def getAllRankerIds(candidates: Seq[CandidateUser]): Seq[RankerId] = candidates.flatMap(_.scores.map(_.scores.flatMap(_.rankerId))).flatten.distinct } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/CandidateShuffle.scala ================================================ package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker import com.twitter.follow_recommendations.common.utils.RandomUtil import scala.util.Random sealed trait CandidateShuffler[T] { def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] } class NoShuffle[T]() extends CandidateShuffler[T] { def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] = input } class RandomShuffler[T]() extends CandidateShuffler[T] { def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] = { seed.map(new Random(_)).getOrElse(Random).shuffle(input) } } trait RankWeightedRandomShuffler[T] extends CandidateShuffler[T] { def rankToWeight(rank: Int): Double def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] = { val candWeights = input.zipWithIndex.map { case (candidate, rank) => (candidate, rankToWeight(rank)) } RandomUtil.weightedRandomShuffle(candWeights, seed.map(new Random(_))).unzip._1 } } class ExponentialShuffler[T]() extends RankWeightedRandomShuffler[T] { def rankToWeight(rank: Int): Double = { 1 / math .pow(rank.toDouble, 2.0) // this function was proved to be effective in previous DDGs } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightMethod.scala ================================================ package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker object WeightMethod extends Enumeration { type WeightMethod = Value val WeightedRandomSampling, WeightedRoundRobin = Value } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceBaseRanker.scala ================================================ package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker import com.twitter.follow_recommendations.common.utils.RandomUtil import com.twitter.follow_recommendations.common.utils.MergeUtil import com.twitter.follow_recommendations.common.utils.Weighted import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.WeightMethod._ import scala.util.Random /** * This ranker selects the next candidate source to select a candidate from. It supports * two kinds of algorithm, WeightedRandomSampling or WeightedRoundRobin. WeightedRandomSampling * pick the next candidate source randomly, WeightedRoundRobin picked the next candidate source * sequentially based on the weight of the candidate source. It is default to WeightedRandomSampling * if no weight method is provided. * * Example usage of this class: * * When use WeightedRandomSampling: * Input candidate sources and their weights are: {{CS1: 3}, {CS2: 2}, {CS3: 5}} * Ranked candidates sequence is not determined because of random sampling. * One possible output candidate sequence is: (CS1_candidate1, CS2_candidate1, CS2_candidate2, * CS3_candidate1, CS3_candidates2, CS3_candidate3, CS1_candidate2, CS1_candidate3, * CS3_candidate4, CS3_candidate5, CS1_candidate4, CS1_candidate5, CS2_candidate6, CS2_candidate3,...) * * When use WeightedRoundRobin: * Input candidate sources and their weights are: {{CS1: 3}, {CS2: 2}, {CS3: 5}} * Output candidate sequence is: (CS1_candidate1, CS1_candidate2, CS1_candidate3, * CS2_candidate1, CS2_candidates2, CS3_candidate1, CS3_candidate2, CS3_candidate3, * CS3_candidate4, CS3_candidate5, CS1_candidate4, CS1_candidate5, CS1_candidate6, CS2_candidate3,...) * * Note: CS1_candidate1 means the first candidate in CS1 candidate source. * * @tparam A candidate source type * @tparam Rec Recommendation type * @param candidateSourceWeights relative weights for different candidate sources */ class WeightedCandidateSourceBaseRanker[A, Rec]( candidateSourceWeights: Map[A, Double], weightMethod: WeightMethod = WeightedRandomSampling, randomSeed: Option[Long]) { /** * Creates a iterator over algorithms and calls next to return a Stream of candidates * * * @param candidateSources the set of candidate sources that are being sampled * @param candidateSourceWeights map of candidate source to weight * @param candidates the map of candidate source to the iterator of its results * @param weightMethod a enum to indict which weight method to use. Two values are supported * currently. When WeightedRandomSampling is set, the next candidate is picked from a candidate * source that is randomly chosen. When WeightedRoundRobin is set, the next candidate is picked * from current candidate source until the number of candidates reaches to the assigned weight of * the candidate source. The next call of this function will return a candidate from the next * candidate source which is after previous candidate source based on the order input * candidate source sequence. * @return stream of candidates */ def stream( candidateSources: Set[A], candidateSourceWeights: Map[A, Double], candidates: Map[A, Iterator[Rec]], weightMethod: WeightMethod = WeightedRandomSampling, random: Option[Random] = None ): Stream[Rec] = { val weightedCandidateSource: Weighted[A] = new Weighted[A] { override def apply(a: A): Double = candidateSourceWeights.getOrElse(a, 0) } /** * Generates a stream of candidates. * * @param candidateSourceIter an iterator over candidate sources returned by the sampling procedure * @return stream of candidates */ def next(candidateSourceIter: Iterator[A]): Stream[Rec] = { val source = candidateSourceIter.next() val it = candidates(source) if (it.hasNext) { val currCand = it.next() currCand #:: next(candidateSourceIter) } else { assert(candidateSources.contains(source), "Selected source is not in candidate sources") // Remove the depleted candidate source and re-sample stream(candidateSources - source, candidateSourceWeights, candidates, weightMethod, random) } } if (candidateSources.isEmpty) Stream.empty else { val candidateSourceSeq = candidateSources.toSeq val candidateSourceIter = if (weightMethod == WeightMethod.WeightedRoundRobin) { MergeUtil.weightedRoundRobin(candidateSourceSeq)(weightedCandidateSource).iterator } else { //default to weighted random sampling if no other weight method is provided RandomUtil .weightedRandomSamplingWithReplacement( candidateSourceSeq, random )(weightedCandidateSource).iterator } next(candidateSourceIter) } } def apply(input: Map[A, TraversableOnce[Rec]]): Stream[Rec] = { stream( input.keySet, candidateSourceWeights, input.map { case (k, v) => k -> v.toIterator }, // cannot do mapValues here, as that only returns a view weightMethod, randomSeed.map(new Random(_)) ) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRanker.scala ================================================ package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker import com.twitter.follow_recommendations.common.base.Ranker import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.rankers.common.DedupCandidates import com.twitter.follow_recommendations.common.rankers.utils.Utils import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams /** * Candidate Ranker that mixes and ranks multiple candidate lists from different candidate sources with the * following steps: * 1) generate a ranked candidate list of each candidate source by sorting and shuffling the candidate list * of the algorithm. * 2) merge the ranked lists generated in 1) into a single list using weighted randomly sampling. * 3) If dedup is required, dedup the output from 2) by candidate id. * * @param basedRanker base ranker * @param shuffleFn the shuffle function that will be used to shuffle each algorithm's sorted candidate list. * @param dedup whether to remove duplicated candidates from the final output. */ class WeightedCandidateSourceRanker[Target <: HasParams]( basedRanker: WeightedCandidateSourceBaseRanker[ CandidateSourceIdentifier, CandidateUser ], shuffleFn: Seq[CandidateUser] => Seq[CandidateUser], dedup: Boolean) extends Ranker[Target, CandidateUser] { val name: String = this.getClass.getSimpleName override def rank(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] = { val scribeRankingInfo: Boolean = target.params(WeightedCandidateSourceRankerParams.ScribeRankingInfoInWeightedRanker) val rankedCands = rankCandidates(group(candidates)) Stitch.value(if (scribeRankingInfo) Utils.addRankingInfo(rankedCands, name) else rankedCands) } private def group( candidates: Seq[CandidateUser] ): Map[CandidateSourceIdentifier, Seq[CandidateUser]] = { val flattened = for { candidate <- candidates identifier <- candidate.getPrimaryCandidateSource } yield (identifier, candidate) flattened.groupBy(_._1).mapValues(_.map(_._2)) } private def rankCandidates( input: Map[CandidateSourceIdentifier, Seq[CandidateUser]] ): Seq[CandidateUser] = { // Sort and shuffle candidates per candidate source. // Note 1: Using map instead mapValue here since mapValue somehow caused infinite loop when used as part of Stream. val sortAndShuffledCandidates = input.map { case (source, candidates) => // Note 2: toList is required here since candidates is a view, and it will result in infinit loop when used as part of Stream. // Note 3: there is no real sorting logic here, it assumes the input is already sorted by candidate sources val sortedCandidates = candidates.toList source -> shuffleFn(sortedCandidates).iterator } val rankedCandidates = basedRanker(sortAndShuffledCandidates) if (dedup) DedupCandidates(rankedCandidates) else rankedCandidates } } object WeightedCandidateSourceRanker { def build[Target <: HasParams]( candidateSourceWeight: Map[CandidateSourceIdentifier, Double], shuffleFn: Seq[CandidateUser] => Seq[CandidateUser] = identity, dedup: Boolean = false, randomSeed: Option[Long] = None ): WeightedCandidateSourceRanker[Target] = { new WeightedCandidateSourceRanker( new WeightedCandidateSourceBaseRanker( candidateSourceWeight, WeightMethod.WeightedRandomSampling, randomSeed = randomSeed), shuffleFn, dedup ) } } object WeightedCandidateSourceRankerWithoutRandomSampling { def build[Target <: HasParams]( candidateSourceWeight: Map[CandidateSourceIdentifier, Double] ): WeightedCandidateSourceRanker[Target] = { new WeightedCandidateSourceRanker( new WeightedCandidateSourceBaseRanker( candidateSourceWeight, WeightMethod.WeightedRoundRobin, randomSeed = None), identity, false, ) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @Singleton class WeightedCandidateSourceRankerFSConfig @Inject() extends FeatureSwitchConfig { override val booleanFSParams: Seq[FSParam[Boolean]] = Seq(WeightedCandidateSourceRankerParams.ScribeRankingInfoInWeightedRanker) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerParams.scala ================================================ package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker import com.twitter.timelines.configapi.FSParam object WeightedCandidateSourceRankerParams { case object ScribeRankingInfoInWeightedRanker extends FSParam[Boolean]("weighted_ranker_scribe_ranking_info", false) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "strato/config/columns/onboarding/userrecs:userrecs-strato-client", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/LowTweepCredFollowStore.scala ================================================ package com.twitter.follow_recommendations.common.stores import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.onboarding.userrecs.TweepCredOnUserClientColumn import javax.inject.Inject import javax.inject.Singleton // Not a candidate source since it's a intermediary. @Singleton class LowTweepCredFollowStore @Inject() (tweepCredOnUserClientColumn: TweepCredOnUserClientColumn) { def getLowTweepCredUsers(target: HasRecentFollowedUserIds): Stitch[Seq[CandidateUser]] = { val newFollowings = target.recentFollowedUserIds.getOrElse(Nil).take(LowTweepCredFollowStore.NumFlockToRetrieve) val validTweepScoreUserIdsStitch: Stitch[Seq[Long]] = Stitch .traverse(newFollowings) { newFollowingUserId => val tweepCredScoreOptStitch = tweepCredOnUserClientColumn.fetcher .fetch(newFollowingUserId) .map(_.v) tweepCredScoreOptStitch.map(_.flatMap(tweepCred => if (tweepCred < LowTweepCredFollowStore.TweepCredThreshold) { Some(newFollowingUserId) } else { None })) }.map(_.flatten) validTweepScoreUserIdsStitch .map(_.map(CandidateUser(_, Some(CandidateUser.DefaultCandidateScore)))) } } object LowTweepCredFollowStore { val NumFlockToRetrieve = 500 val TweepCredThreshold = 40 } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/DedupTransform.scala ================================================ package com.twitter.follow_recommendations.common.transforms.dedup import com.twitter.follow_recommendations.common.base.Transform import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.stitch.Stitch import scala.collection.mutable class DedupTransform[Request, Candidate <: UniversalNoun[Long]]() extends Transform[Request, Candidate] { override def transform(target: Request, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] = { val seen = mutable.HashSet[Long]() Stitch.value(candidates.filter(candidate => seen.add(candidate.id))) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "configapi/configapi-core", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", "snowflake/src/main/scala/com/twitter/snowflake/id", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/ModifySocialProofTransform.scala ================================================ package com.twitter.follow_recommendations.common.transforms.modify_social_proof import com.twitter.conversions.DurationOps._ import com.twitter.decider.Decider import com.twitter.decider.RandomRecipient import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.util.DefaultTimer import com.twitter.follow_recommendations.common.base.GatedTransform import com.twitter.follow_recommendations.common.clients.graph_feature_service.GraphFeatureServiceClient import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FollowProof import com.twitter.follow_recommendations.configapi.deciders.DeciderKey import com.twitter.graph_feature_service.thriftscala.EdgeType import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.util.logging.Logging import com.twitter.util.Duration import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton object ModifySocialProof { val GfsLagDuration: Duration = 14.days val GfsIntersectionIds: Int = 3 val SgsIntersectionIds: Int = 10 val LeftEdgeTypes: Set[EdgeType] = Set(EdgeType.Following) val RightEdgeTypes: Set[EdgeType] = Set(EdgeType.FollowedBy) /** * Given the intersection ID's for a particular candidate, update the candidate's social proof * @param candidate candidate object * @param followProof follow proof to be added (includes id's and count) * @param stats stats for tracking * @return updated candidate object */ def addIntersectionIdsToCandidate( candidate: CandidateUser, followProof: FollowProof, stats: StatsReceiver ): CandidateUser = { // create updated set of social proof val updatedFollowedByOpt = candidate.followedBy match { case Some(existingFollowedBy) => Some((followProof.followedBy ++ existingFollowedBy).distinct) case None if followProof.followedBy.nonEmpty => Some(followProof.followedBy.distinct) case _ => None } val updatedFollowProof = updatedFollowedByOpt.map { updatedFollowedBy => val updatedCount = followProof.numIds.max(updatedFollowedBy.size) // track stats val numSocialProofAdded = updatedFollowedBy.size - candidate.followedBy.size addCandidatesWithSocialContextCountStat(stats, numSocialProofAdded) FollowProof(updatedFollowedBy, updatedCount) } candidate.setFollowProof(updatedFollowProof) } private def addCandidatesWithSocialContextCountStat( statsReceiver: StatsReceiver, count: Int ): Unit = { if (count > 3) { statsReceiver.counter("4_and_more").incr() } else { statsReceiver.counter(count.toString).incr() } } } /** * This class makes a request to gfs/sgs for hydrating additional social proof on each of the * provided candidates. */ @Singleton class ModifySocialProof @Inject() ( gfsClient: GraphFeatureServiceClient, socialGraphClient: SocialGraphClient, statsReceiver: StatsReceiver, decider: Decider = Decider.True) extends Logging { import ModifySocialProof._ private val stats = statsReceiver.scope(this.getClass.getSimpleName) private val addedStats = stats.scope("num_social_proof_added_per_candidate") private val gfsStats = stats.scope("graph_feature_service") private val sgsStats = stats.scope("social_graph_service") private val previousProofEmptyCounter = stats.counter("previous_proof_empty") private val emptyFollowProofCounter = stats.counter("empty_followed_proof") /** * For each candidate provided, we get the intersectionIds between the user and the candidate, * appending the unique results to the social proof (followedBy field) if not already previously * seen we query GFS for all users, except for cases specified via the mustCallSgs field or for * very new users, who would not have any data in GFS, due to the lag duration of the service's * processing. this is determined by GfsLagDuration * @param userId id of the target user whom we provide recommendations for * @param candidates list of candidates * @param intersectionIdsNum if provided, determines the maximum number of accounts we want to be hydrated for social proof * @param mustCallSgs Determines if we should query SGS regardless of user age or not. * @return list of candidates updated with additional social proof */ def hydrateSocialProof( userId: Long, candidates: Seq[CandidateUser], intersectionIdsNum: Option[Int] = None, mustCallSgs: Boolean = false, callSgsCachedColumn: Boolean = false, gfsLagDuration: Duration = GfsLagDuration, gfsIntersectionIds: Int = GfsIntersectionIds, sgsIntersectionIds: Int = SgsIntersectionIds, ): Stitch[Seq[CandidateUser]] = { addCandidatesWithSocialContextCountStat( stats.scope("social_context_count_before_hydration"), candidates.count(_.followedBy.isDefined) ) val candidateIds = candidates.map(_.id) val userAgeOpt = SnowflakeId.timeFromIdOpt(userId).map(Time.now - _) // this decider gate is used to determine what % of requests is allowed to call // Graph Feature Service. this is useful for ramping down requests to Graph Feature Service // when necessary val deciderKey: String = DeciderKey.EnableGraphFeatureServiceRequests.toString val enableGfsRequests: Boolean = decider.isAvailable(deciderKey, Some(RandomRecipient)) // if new query sgs val (candidateToIntersectionIdsMapFut, isGfs) = if (!enableGfsRequests || mustCallSgs || userAgeOpt.exists(_ < gfsLagDuration)) { ( if (callSgsCachedColumn) socialGraphClient.getIntersectionsFromCachedColumn( userId, candidateIds, intersectionIdsNum.getOrElse(sgsIntersectionIds) ) else socialGraphClient.getIntersections( userId, candidateIds, intersectionIdsNum.getOrElse(sgsIntersectionIds)), false) } else { ( gfsClient.getIntersections( userId, candidateIds, intersectionIdsNum.getOrElse(gfsIntersectionIds)), true) } val finalCandidates = candidateToIntersectionIdsMapFut .map { candidateToIntersectionIdsMap => { previousProofEmptyCounter.incr(candidates.count(_.followedBy.exists(_.isEmpty))) candidates.map { candidate => addIntersectionIdsToCandidate( candidate, candidateToIntersectionIdsMap.getOrElse(candidate.id, FollowProof(Seq.empty, 0)), addedStats) } } } .within(250.milliseconds)(DefaultTimer) .rescue { case e: Exception => error(e.getMessage) if (isGfs) { gfsStats.scope("rescued").counter(e.getClass.getSimpleName).incr() } else { sgsStats.scope("rescued").counter(e.getClass.getSimpleName).incr() } Stitch.value(candidates) } finalCandidates.onSuccess { candidatesSeq => emptyFollowProofCounter.incr(candidatesSeq.count(_.followedBy.exists(_.isEmpty))) addCandidatesWithSocialContextCountStat( stats.scope("social_context_count_after_hydration"), candidatesSeq.count(_.followedBy.isDefined) ) } } } /** * This transform uses ModifySocialProof (which makes a request to gfs/sgs) for hydrating additional * social proof on each of the provided candidates. */ @Singleton class ModifySocialProofTransform @Inject() (modifySocialProof: ModifySocialProof) extends GatedTransform[HasClientContext with HasParams, CandidateUser] with Logging { override def transform( target: HasClientContext with HasParams, candidates: Seq[CandidateUser] ): Stitch[Seq[CandidateUser]] = target.getOptionalUserId .map(modifySocialProof.hydrateSocialProof(_, candidates)).getOrElse(Stitch.value(candidates)) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/RemoveAccountProofTransform.scala ================================================ package com.twitter.follow_recommendations.common.transforms.modify_social_proof import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.GatedTransform import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import javax.inject.Inject import javax.inject.Singleton @Singleton class RemoveAccountProofTransform @Inject() (statsReceiver: StatsReceiver) extends GatedTransform[HasClientContext with HasParams, CandidateUser] { private val stats = statsReceiver.scope(this.getClass.getSimpleName) private val removedProofsCounter = stats.counter("num_removed_proofs") override def transform( target: HasClientContext with HasParams, items: Seq[CandidateUser] ): Stitch[Seq[CandidateUser]] = Stitch.value(items.map { candidate => removedProofsCounter.incr() candidate.copy(reason = None) }) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/RandomRankerIdTransform.scala ================================================ package com.twitter.follow_recommendations.common.transforms.ranker_id import com.google.inject.Inject import com.google.inject.Singleton import com.twitter.follow_recommendations.common.base.GatedTransform import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.Score import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams /** * This class appends each candidate's rankerIds with the RandomRankerId. * This is primarily for determining if a candidate was generated via random shuffling. */ @Singleton class RandomRankerIdTransform @Inject() () extends GatedTransform[HasParams, CandidateUser] { override def transform( target: HasParams, candidates: Seq[CandidateUser] ): Stitch[Seq[CandidateUser]] = { Stitch.value(candidates.map(_.addScore(Score.RandomScore))) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/AddRecommendationFlowIdentifierTransform.scala ================================================ package com.twitter.follow_recommendations.common.transforms.recommendation_flow_identifier import com.google.inject.Inject import com.twitter.follow_recommendations.common.base.Transform import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasRecommendationFlowIdentifier import com.twitter.stitch.Stitch class AddRecommendationFlowIdentifierTransform @Inject() extends Transform[HasRecommendationFlowIdentifier, CandidateUser] { override def transform( target: HasRecommendationFlowIdentifier, items: Seq[CandidateUser] ): Stitch[Seq[CandidateUser]] = { Stitch.value(items.map { candidateUser => candidateUser.copy(recommendationFlowIdentifier = target.recommendationFlowIdentifier) }) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/TrackingTokenTransform.scala ================================================ package com.twitter.follow_recommendations.common.transforms.tracking_token import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.Transform import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.common.models.Session import com.twitter.follow_recommendations.common.models.TrackingToken import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap import com.twitter.hermit.model.Algorithm import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.stitch.Stitch import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton /** * This transform adds the tracking token for all candidates * Since this happens in the same request, we use the same trace-id for all candidates * There are no RPC calls in this transform so it's safe to chain it with `andThen` at the end of * all other product-specific transforms */ @Singleton class TrackingTokenTransform @Inject() (baseStatsReceiver: StatsReceiver) extends Transform[HasDisplayLocation with HasClientContext, CandidateUser] with Logging { def profileResults( target: HasDisplayLocation with HasClientContext, candidates: Seq[CandidateUser] ) = { // Metrics to track # results per candidate source val stats = baseStatsReceiver.scope(target.displayLocation.toString + "/final_results") stats.stat("total").add(candidates.size) stats.counter(target.displayLocation.toString).incr() val flattenedCandidates: Seq[(CandidateSourceIdentifier, CandidateUser)] = for { candidate <- candidates identifier <- candidate.getPrimaryCandidateSource } yield (identifier, candidate) val candidatesGroupedBySource: Map[CandidateSourceIdentifier, Seq[CandidateUser]] = flattenedCandidates.groupBy(_._1).mapValues(_.map(_._2)) candidatesGroupedBySource map { case (source, candidates) => stats.stat(source.name).add(candidates.size) } } override def transform( target: HasDisplayLocation with HasClientContext, candidates: Seq[CandidateUser] ): Stitch[Seq[CandidateUser]] = { profileResults(target, candidates) Stitch.value( target.getOptionalUserId .map { _ => candidates.map { candidate => val token = Some(TrackingToken( sessionId = Session.getSessionId, displayLocation = Some(target.displayLocation), controllerData = None, algorithmId = candidate.userCandidateSourceDetails.flatMap(_.primaryCandidateSource .flatMap { identifier => Algorithm.withNameOpt(identifier.name).flatMap(AlgorithmToFeedbackTokenMap.get) }) )) candidate.copy(trackingToken = token) } }.getOrElse(candidates)) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransform.scala ================================================ package com.twitter.follow_recommendations.common.transforms.weighted_sampling import com.twitter.follow_recommendations.common.base.GatedTransform import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDebugOptions import com.twitter.follow_recommendations.common.models.Score import com.twitter.follow_recommendations.common.models.Scores import com.twitter.follow_recommendations.common.rankers.common.RankerId import com.twitter.follow_recommendations.common.rankers.utils.Utils import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import javax.inject.Inject import javax.inject.Singleton @Singleton class SamplingTransform @Inject() () extends GatedTransform[HasClientContext with HasParams with HasDebugOptions, CandidateUser] { val name: String = this.getClass.getSimpleName /* Description: This function takes in a set of candidate users and ranks them for a who-to-follow request by sampling from the Placket-Luce distribution (https://cran.rstudio.com/web/packages/PlackettLuce/vignettes/Overview.html) with a three variations. The first variation is that the scores of the candidates are multiplied by multiplicativeFactor before sampling. The second variation is that the scores are exponentiated before sampling. The third variation is that depending on how many who-to-follow positions are being requested, the first k positions are reserved for the candidates with the highest scores (and they are sorted in decreasing order of score) and the remaining positions are sampled from a Placket-Luce. We use the efficient algorithm proposed in this blog https://medium.com/swlh/going-old-school-designing-algorithms-for-fast-weighted-sampling-in-production-c48fc1f40051 to sample from a Plackett-Luce. Because of numerical stability reasons, before sampling from this distribution, (1) we subtract off the maximum score from all the scores and (2) if after this subtraction and multiplication by the multiplicative factor the resulting score is <= -10, we force the candidate's transformed score under the above algorithm to be 0 (so r^(1/w) = 0) where r is a random number and w is the transformed score. inputs: - target: HasClientContext (WTF request) - candidates: sequence of CandidateUsers (users that need to be ranked from a who-to-follow request) each of which has a score inputs accessed through feature switches, i.e. through target.params (see the following file: "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/ transforms/weighted_sampling/SamplingTransformParams.scala"): - topKFixed: the first k positions of the who-to-follow ranking correspond to the users with the k highest scores and are not sampled from the Placket-Luce distribution - multiplicativeFactor: multiplicativeFactor is used to transform the scores of each candidate by multiplying that user's score by multiplicativeFactor output: - Sequence of CandidateUser whose order represents the ranking of users in a who-to-follow request This ranking is sampled from a Placket-Luce distribution. */ override def transform( target: HasClientContext with HasParams with HasDebugOptions, candidates: Seq[CandidateUser] ): Stitch[Seq[CandidateUser]] = { // the first k positions of the who-to-follow ranking correspond to the users with the k // highest scores and are not sampled from the Placket-Luce distribution val topKFixed = target.params(SamplingTransformParams.TopKFixed) // multiplicativeFactor is used to transform the scores of each candidate by // multiplying that user's score by multiplicativeFactor val multiplicativeFactor = target.params(SamplingTransformParams.MultiplicativeFactor) // sort candidates by their score val candidatesSorted = candidates.sortBy(-1 * _.score.getOrElse(0.0)) // pick the top K candidates by score and the remaining candidates val (topKFixedCandidates, candidatesOutsideOfTopK) = candidatesSorted.zipWithIndex.partition { case (value, index) => index < topKFixed } val randomNumGenerator = new scala.util.Random(target.getRandomizationSeed.getOrElse(System.currentTimeMillis)) // we need to subtract the maximum score off the scores for numerical stability reasons // subtracting the max score off does not effect the underlying distribution we are sampling // the candidates from // we need the if statement since you cannot take the max of an empty sequence val maximum_score = if (candidatesOutsideOfTopK.nonEmpty) { candidatesOutsideOfTopK.map(x => x._1.score.getOrElse(0.0)).max } else { 0.0 } // for candidates in candidatesOutsideOfTopK, we transform their score by subtracting off // maximum_score and then multiply by multiplicativeFactor val candidatesOutsideOfTopKTransformedScore = candidatesOutsideOfTopK.map(x => (x._1, multiplicativeFactor * (x._1.score.getOrElse(0.0) - maximum_score))) // for each candidate with score transformed and clip score w, sample a random number r, // create a new score r^(1/w) and sort the candidates to get the final ranking. // for numerical stability reasons if the score is <=-10, we force r^(1/w) = 0. // this samples the candidates from the modified Plackett-Luce distribution. See // https://medium.com/swlh/going-old-school-designing-algorithms-for-fast-weighted-sampling-in-production-c48fc1f40051 val candidatesOutsideOfTopKSampled = candidatesOutsideOfTopKTransformedScore .map(x => ( x._1, if (x._2 <= -10.0) 0.0 else scala.math.pow( randomNumGenerator.nextFloat(), 1 / (scala.math .exp(x._2))))).sortBy(-1 * _._2) val topKCandidates: Seq[CandidateUser] = topKFixedCandidates.map(_._1) val scribeRankingInfo: Boolean = target.params(SamplingTransformParams.ScribeRankingInfoInSamplingTransform) val transformedCandidates: Seq[CandidateUser] = if (scribeRankingInfo) { val topKCandidatesWithRankingInfo: Seq[CandidateUser] = Utils.addRankingInfo(topKCandidates, name) val candidatesOutsideOfTopKSampledWithRankingInfo: Seq[CandidateUser] = candidatesOutsideOfTopKSampled.zipWithIndex.map { case ((candidate, score), rank) => val newScore = Seq(Score(score, Some(RankerId.PlacketLuceSamplingTransformer))) val newScores: Option[Scores] = candidate.scores .map { scores => scores.copy(scores = scores.scores ++ newScore) }.orElse(Some(Scores(newScore, Some(RankerId.PlacketLuceSamplingTransformer)))) val globalRank = rank + topKFixed + 1 candidate.addInfoPerRankingStage(name, newScores, globalRank) } topKCandidatesWithRankingInfo ++ candidatesOutsideOfTopKSampledWithRankingInfo } else { topKCandidates ++ candidatesOutsideOfTopKSampled.map(_._1) } Stitch.value(transformedCandidates) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformFSConfig.scala ================================================ package com.twitter.follow_recommendations.common.transforms.weighted_sampling import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @Singleton class SamplingTransformFSConfig @Inject() () extends FeatureSwitchConfig { override val intFSParams: Seq[FSBoundedParam[Int]] = Seq(SamplingTransformParams.TopKFixed) override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( SamplingTransformParams.MultiplicativeFactor) override val booleanFSParams: Seq[FSParam[Boolean]] = Seq( SamplingTransformParams.ScribeRankingInfoInSamplingTransform) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformParams.scala ================================================ package com.twitter.follow_recommendations.common.transforms.weighted_sampling import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam object SamplingTransformParams { case object TopKFixed // indicates how many of the fisrt K who-to-follow recommendations are reserved for the candidates with largest K CandidateUser.score where these candidates are sorted in decreasing order of score extends FSBoundedParam[Int]( name = "post_nux_ml_flow_weighted_sampling_top_k_fixed", default = 0, min = 0, max = 100) case object MultiplicativeFactor // CandidateUser.score gets transformed to multiplicativeFactor*CandidateUser.score before sampling from the Plackett-Luce distribution extends FSBoundedParam[Double]( name = "post_nux_ml_flow_weighted_sampling_multiplicative_factor", default = 1.0, min = -1000.0, max = 1000.0) case object ScribeRankingInfoInSamplingTransform extends FSParam[Boolean]("sampling_transform_scribe_ranking_info", false) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/BUILD ================================================ scala_library( platform = "java8", tags = ["bazel-compatible"], dependencies = [ "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", "stitch/stitch-core", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/CollectionUtil.scala ================================================ package com.twitter.follow_recommendations.common.utils object CollectionUtil { /** * Transposes a sequence of sequences. As opposed to the Scala collection library version * of transpose, the sequences do not have to have the same length. * * Example: * transpose(immutable.Seq(immutable.Seq(1,2,3), immutable.Seq(4,5), immutable.Seq(6,7))) * => immutable.Seq(immutable.Seq(1, 4, 6), immutable.Seq(2, 5, 7), immutable.Seq(3)) * * @param seq a sequence of sequences * @tparam A the type of elements in the seq * @return the transposed sequence of sequences */ def transposeLazy[A](seq: Seq[Seq[A]]): Stream[Seq[A]] = seq.filter(_.nonEmpty) match { case Nil => Stream.empty case ys => ys.map(_.head) #:: transposeLazy(ys.map(_.tail)) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/DisplayLocationProductConverterUtil.scala ================================================ package com.twitter.follow_recommendations.common.utils import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.follow_recommendations.common.models.Product import com.twitter.product_mixer.core.model.marshalling.request.Product object DisplayLocationProductConverterUtil { def productToDisplayLocation(product: Product): DisplayLocation = { product match { case Product.MagicRecs => DisplayLocation.MagicRecs case _ => throw UnconvertibleProductMixerProductException( s"Cannot convert Product Mixer Product ${product.identifier.name} into a FRS DisplayLocation.") } } def displayLocationToProduct(displayLocation: DisplayLocation): Product = { displayLocation match { case DisplayLocation.MagicRecs => Product.MagicRecs case _ => throw UnconvertibleProductMixerProductException( s"Cannot convert DisplayLocation ${displayLocation.toFsName} into a Product Mixer Product.") } } } case class UnconvertibleProductMixerProductException(message: String) extends Exception(message) ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/MergeUtil.scala ================================================ package com.twitter.follow_recommendations.common.utils object MergeUtil { /** * Takes a seq of items which have weights. Returns an infinite stream of each item * by their weights. All weights need to be greater than or equal to zero. In addition, * the sum of weights should be greater than zero. * * Example usage of this function: * Input weighted Item {{CS1, 3}, {CS2, 2}, {CS3, 5}} * Output stream: (CS1, CS1, CS1, CS2, CS2, CS3, CS3, CS3, CS3, CS3, CS1, CS1, CS1, CS2,...} * * @param items items * @param weighted provides weights for items * @tparam T type of item * * @return Stream of Ts */ def weightedRoundRobin[T]( items: Seq[T] )( implicit weighted: Weighted[T] ): Stream[T] = { if (items.isEmpty) { Stream.empty } else { val weights = items.map { i => weighted(i) } assert( weights.forall { _ >= 0 }, "Negative weight exists for sampling") val cumulativeWeight = weights.scanLeft(0.0)(_ + _).tail assert(cumulativeWeight.last > 0, "Sum of the sampling weights is not positive") var weightIdx = 0 var weight = 0 def next(): Stream[T] = { val tmpIdx = weightIdx weight = weight + 1 weight = if (weight >= weights(weightIdx)) 0 else weight weightIdx = if (weight == 0) weightIdx + 1 else weightIdx weightIdx = if (weightIdx == weights.length) 0 else weightIdx items(tmpIdx) #:: next() } next() } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RandomUtil.scala ================================================ package com.twitter.follow_recommendations.common.utils import scala.util.Random object RandomUtil { /** * Takes a seq of items which have weights. Returns an infinite stream that is * sampled with replacement using the weights for each item. All weights need * to be greater than or equal to zero. In addition, the sum of weights * should be greater than zero. * * @param items items * @param weighted provides weights for items * @tparam T type of item * @return Stream of Ts */ def weightedRandomSamplingWithReplacement[T]( items: Seq[T], random: Option[Random] = None )( implicit weighted: Weighted[T] ): Stream[T] = { if (items.isEmpty) { Stream.empty } else { val weights = items.map { i => weighted(i) } assert(weights.forall { _ >= 0 }, "Negative weight exists for sampling") val cumulativeWeight = weights.scanLeft(0.0)(_ + _).tail assert(cumulativeWeight.last > 0, "Sum of the sampling weights is not positive") val cumulativeProbability = cumulativeWeight map (_ / cumulativeWeight.last) def next(): Stream[T] = { val rand = random.getOrElse(Random).nextDouble() val idx = cumulativeProbability.indexWhere(_ >= rand) items(if (idx == -1) items.length - 1 else idx) #:: next() } next() } } /** * Takes a seq of items and their weights. Returns a lazy weighted shuffle of * the elements in the list. All weights should be greater than zero. * * @param items items * @param weighted provides weights for items * @tparam T type of item * @return Stream of Ts */ def weightedRandomShuffle[T]( items: Seq[T], random: Option[Random] = None )( implicit weighted: Weighted[T] ): Stream[T] = { assert(items.forall { i => weighted(i) > 0 }, "Non-positive weight exists for shuffling") def next(it: Seq[T]): Stream[T] = { if (it.isEmpty) Stream.empty else { val cumulativeWeight = it.scanLeft(0.0)((acc: Double, curr: T) => acc + weighted(curr)).tail val cutoff = random.getOrElse(Random).nextDouble() * cumulativeWeight.last val idx = cumulativeWeight.indexWhere(_ >= cutoff) val (left, right) = it.splitAt(idx) it(if (idx == -1) it.size - 1 else idx) #:: next(left ++ right.drop(1)) } } next(items) } /** * Takes a seq of items and a weight function, returns a lazy weighted shuffle of * the elements in the list.The weight function is based on the rank of the element * in the original lst. * @param items * @param rankToWeight * @param random * @tparam T * @return */ def weightedRandomShuffleByRank[T]( items: Seq[T], rankToWeight: Int => Double, random: Option[Random] = None ): Stream[T] = { val candWeights = items.zipWithIndex.map { case (item, rank) => (item, rankToWeight(rank)) } RandomUtil.weightedRandomShuffle(candWeights, random).map(_._1) } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RescueWithStatsUtils.scala ================================================ package com.twitter.follow_recommendations.common.utils import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.StatsUtil import com.twitter.stitch.Stitch import com.twitter.util.Duration import com.twitter.util.TimeoutException object RescueWithStatsUtils { def rescueWithStats[T]( s: Stitch[Seq[T]], stats: StatsReceiver, source: String ): Stitch[Seq[T]] = { StatsUtil.profileStitchSeqResults(s, stats.scope(source)).rescue { case _: Exception => Stitch.Nil } } def rescueOptionalWithStats[T]( s: Stitch[Option[T]], stats: StatsReceiver, source: String ): Stitch[Option[T]] = { StatsUtil.profileStitchOptionalResults(s, stats.scope(source)).rescue { case _: Exception => Stitch.None } } def rescueWithStatsWithin[T]( s: Stitch[Seq[T]], stats: StatsReceiver, source: String, timeout: Duration ): Stitch[Seq[T]] = { val hydratedScopeSource = stats.scope(source) StatsUtil .profileStitchSeqResults( s.within(timeout)(com.twitter.finagle.util.DefaultTimer), hydratedScopeSource) .rescue { case _: TimeoutException => hydratedScopeSource.counter("timeout").incr() Stitch.Nil case _: Exception => hydratedScopeSource.counter("exception").incr() Stitch.Nil } } } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/UserSignupUtil.scala ================================================ package com.twitter.follow_recommendations.common.utils import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.snowflake.id.SnowflakeId import com.twitter.util.Duration import com.twitter.util.Time object UserSignupUtil { def signupTime(hasClientContext: HasClientContext): Option[Time] = hasClientContext.clientContext.userId.flatMap(SnowflakeId.timeFromIdOpt) def userSignupAge(hasClientContext: HasClientContext): Option[Duration] = signupTime(hasClientContext).map(Time.now - _) } ================================================ FILE: follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/Weighted.scala ================================================ package com.twitter.follow_recommendations.common.utils /** * Typeclass for any Recommendation type that has a weight * */ trait Weighted[-Rec] { def apply(rec: Rec): Double } object Weighted { implicit object WeightedTuple extends Weighted[(_, Double)] { override def apply(rec: (_, Double)): Double = rec._2 } def fromFunction[Rec](f: Rec => Double): Weighted[Rec] = { new Weighted[Rec] { override def apply(rec: Rec): Double = f(rec) } } } ================================================ FILE: follow-recommendations-service/server/src/main/resources/BUILD ================================================ resources( sources = [ "*.tsv", "*.xml", "**/*", "config/*.yml", ], ) # Created for Bazel compatibility. # In Bazel, loose files must be part of a target to be included into a bundle. files( name = "frs_resources", sources = [ "*.tsv", "*.xml", "*.yml", "**/*", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/resources/config/decider.yml ================================================ enable_recommendations: comment: Proportion of requests where we return an actual response as part. Decreasing the value will increase the portion of empty responses (in order to disable the service) as part of the graceful degradation. default_availability: 10000 enable_score_user_candidates: comment: Proportion of requests where score user candidates from the scoreUserCandidates endpoint default_availability: 10000 enable_profile_sidebar_product: comment: Proportion of requests where we return an actual response for profile sidebar product default_availability: 10000 enable_magic_recs_product: comment: Proportion of requests where we return an actual response for magic recs product default_availability: 10000 enable_rux_landing_page_product: comment: Proportion of requests where we return an actual response for rux landing page product default_availability: 10000 enable_rux_pymk_product: comment: Proportion of requests where we return an actual response for rux pymk product default_availability: 10000 enable_profile_bonus_follow_product: comment: Proportion of requests where we return an actual response for profile bonus follow product default_availability: 10000 enable_election_explore_wtf_product: comment: Proportion of requests where we return an actual response for election explore wtf product default_availability: 10000 enable_cluster_follow_product: comment: Proportion of requests where we return an actual response for cluster follow product default_availability: 10000 enable_home_timeline_product: comment: Proportion of requests where we return an actual response for htl wtf product default_availability: 10000 enable_htl_bonus_follow_product: comment: Proportion of requests where we return an actual response for htl bonus follow product default_availability: 10000 enable_explore_tab_product: comment: Proportion of requests where we return an actual response for explore tab product default_availability: 10000 enable_sidebar_product: comment: Proportion of requests where we return an actual response for sidebar product default_availability: 10000 enable_campaign_form_product: comment: Proportion of requests where we return an actual response for campaign form product default_availability: 10000 enable_reactive_follow_product: comment: Proportion of requests where we return an actual response for reactive follow product default_availability: 10000 enable_nux_pymk_product: comment: Proportion of requests where we return an actual response for nux pymk product default_availability: 10000 enable_nux_interests_product: comment: Proportion of requests where we return an actual response for nux interests product default_availability: 10000 enable_nux_topic_bonus_follow_product: comment: Proportion of requests where we return an actual response for nux topic-based bonus follow product default_availability: 10000 enable_india_covid19_curated_accounts_wtf_product: comment: Proportion of requests where we return an actual response for india covid19 curated accounts wtf product default_availability: 10000 enable_ab_upload_product: comment: Proportion of requests where we return an actual response for the address book upload product default_availability: 10000 enable_people_plus_plus_product: comment: Proportion of requests where we return an actual response for the PeoplePlusPlus/Connect Tab product default_availability: 10000 enable_tweet_notification_recs_product: comment: Proportion of requests where we return an actual response for the Tweet Notification Recommendations product default_availability: 10000 enable_profile_device_follow_product: comment: Proportion of requests where we return an actual response for the ProfileDeviceFollow product default_availability: 10000 enable_diffy_module_dark_reading: comment: Percentage of dark read traffic routed to diffy thrift default_availability: 0 enable_recos_backfill_product: comment: Proportion of requests where we return an actual response for the RecosBackfill product default_availability: 10000 enable_post_nux_follow_task_product: comment: Proportion of requests where we return an actual response for post NUX follow task product default_availability: 10000 enable_curated_space_hosts_product: comment: Proportion of requests where we return an actual response for curated space hosts product default_availability: 10000 enable_nux_geo_category_product: comment: Proportion of requests where we return an actual response for nux geo category product default_availability: 10000 enable_nux_interests_category_product: comment: Proportion of requests where we return an actual response for nux interests category product default_availability: 10000 enable_nux_pymk_category_product: comment: Proportion of requests where we return an actual response for nux pymk category product default_availability: 10000 enable_home_timeline_tweet_recs_product: comment: Proportion of requests where we return an actual response for the Home Timeline Tweet Recs product default_availability: 10000 enable_htl_bulk_friend_follows_product: comment: Proportion of requests where we return an actual response for the HTL bulk friend follows product default_availability: 10000 enable_nux_auto_follow_product: comment: Proportion of requests where we return an actual response for the NUX auto follow product default_availability: 10000 enable_search_bonus_follow_product: comment: Proportion of requests where we return an actual response for search bonus follow product default_availability: 10000 enable_fetch_user_in_request_builder: comment: Proportion of requests where we fetch user object from gizmoduck in request builder default_availability: 0 enable_product_mixer_magic_recs_product: comment: Proportion of requests where we enable the product mixer magic recs product default_availability: 10000 enable_home_timeline_reverse_chron_product: comment: Proportion of requests where we return an actual response for Home timeline reverse chron product default_availability: 10000 enable_product_mixer_pipeline_magic_recs_dark_read: comment: Compare product mixer pipeline responses to current FRS pipeline responses for Magic Recs default_availability: 0 enable_experimental_caching: comment: Proportion of requests we use experimental caching for data caching default_availability: 0 enable_distributed_caching: comment: Proportion of requests we use a distributed cache cluster for data caching default_availability: 10000 enable_gizmoduck_caching: comment: Proportion of requests we use a distributed cache cluster for data caching in Gizmoduck default_availability: 10000 enable_traffic_dark_reading: comment: Proportion of requests where we replicate the request for traffic dark reading default_availability: 0 enable_graph_feature_service_requests: comment: Proportion of requests where we allow request calls to Graph Feature Service default_availability: 10000 ================================================ FILE: follow-recommendations-service/server/src/main/resources/logback.xml ================================================ true ${log.service.output} ${log.service.output}.%i 1 5 50MB %date %.-3level ${DEFAULT_SERVICE_PATTERN}%n ${log.access.output} ${log.access.output}.%i 1 5 50MB ${DEFAULT_ACCESS_PATTERN}%n true ${log.lens.category} ${log.lens.index} ${log.lens.tag}/service %msg true ${log.lens.category} ${log.lens.index} ${log.lens.tag}/access %msg ${async_queue_size} ${async_max_flush_time} ${async_queue_size} ${async_max_flush_time} ${async_queue_size} ${async_max_flush_time} ${async_queue_size} ${async_max_flush_time} ================================================ FILE: follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/epModel ================================================ # OWNER = jdeng # Date = 20141223_153423 # Training Size = 16744473 # Testing Size = 16767335 # trained with ElasticNetCV alpha=0.05 cv_folds=5 best_lambda=1.0E-7 # num base features: 10 # num nonzero weights: 30 {bias:-5.67151,featureMetadataMap:["fwd_email":{metadata:{featureWeight:{weight:2.47389}}},"rev_phone":{metadata:{featureWeight:{weight:1.88433}}},"mutual_follow_path":{metadata:{featureWeight:{intervalWeights:[{left:47.0,weight:6.31809},{left:11.0,right:16.0,weight:4.52959},{left:31.0,right:47.0,weight:5.7101},{right:2.0,weight:0.383515},{left:24.0,right:31.0,weight:5.26515},{left:3.0,right:4.0,weight:2.91751},{left:2.0,right:3.0,weight:2.22851},{left:4.0,right:5.0,weight:3.28515},{left:8.0,right:11.0,weight:4.14731},{left:5.0,right:8.0,weight:3.73588},{left:16.0,right:24.0,weight:4.90908}]}}},"fwd_phone":{metadata:{featureWeight:{weight:2.07327}}},"fwd_email_path":{metadata:{featureWeight:{weight:0.961773}}},"rev_phone_path":{metadata:{featureWeight:{weight:0.354484}}},"low_tweepcred_follow_path":{metadata:{featureWeight:{intervalWeights:[{left:4.0,right:5.0,weight:0.177209},{left:7.0,right:8.0,weight:0.12378},{left:3.0,right:4.0,weight:0.197566},{left:5.0,right:6.0,weight:0.15867},{left:2.0,right:3.0,weight:0.196539},{right:2.0,weight:0.1805},{left:75.0,weight:-0.424598},{left:6.0,right:7.0,weight:0.143698},{left:10.0,right:13.0,weight:0.0458502},{left:8.0,right:10.0,weight:0.0919314},{left:13.0,right:75.0,weight:-0.111484}]}}},"rev_email_path":{metadata:{featureWeight:{weight:0.654451}}},"rev_email":{metadata:{featureWeight:{weight:2.33859}}},"fwd_phone_path":{metadata:{featureWeight:{weight:0.210418}}}]} ================================================ FILE: follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/trainingConfig ================================================ {input:{context:"discover.prod",startDateTime:"",endDateTime:"",trainingFeatures:["STP_FEATURES":["fwd_email","mutual_follow_path","fwd_email_path","rev_phone_path","low_tweepcred_follow_path","rev_phone","fwd_phone","rev_email_path","rev_email","fwd_phone_path"]],engagementActions:["click","favorite","open_link","open","send_tweet","send_reply","retweet","reply","profile_click","follow"],impressionActions:["discard","results","impression"],dataFormat:1,dataPath:"",isLabeled:0},sample:{positiveSampleRatio:1.0,negativeSampleRatio:1.0,sampleType:1},split:{trainingDataSplitSize:0.5,testingDataSplitSize:0.5,splitType:2},transform:{},filter:{featureOptions:[]},join:{engagementRules:["discover"],contentIdType:"tweet",groupBucketSize:3600000},discretize:{}} ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/slf4j:slf4j-api", "finagle/finagle-http/src/main/scala", "finagle/finagle-thriftmux/src/main/scala", "finatra-internal/decider/src/main/scala", "finatra-internal/international/src/main/scala/com/twitter/finatra/international/modules", "finatra-internal/mtls-http/src/main/scala", "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/http-core/src/main/java/com/twitter/finatra/http", "finatra/inject/inject-app/src/main/scala", "finatra/inject/inject-core/src/main/scala", "finatra/inject/inject-server/src/main/scala", "finatra/inject/inject-thrift-client", "finatra/jackson/src/main/scala/com/twitter/finatra/jackson/modules", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", "follow-recommendations-service/server/src/main/resources", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions", "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "geoduck/service/src/main/scala/com/twitter/geoduck/service/common/clientmodules", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", "twitter-server/server/src/main/scala", "util/util-app/src/main/scala", "util/util-core:scala", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/FollowRecommendationsServiceThriftServer.scala ================================================ package com.twitter.follow_recommendations import com.google.inject.Module import com.twitter.finagle.ThriftMux import com.twitter.finatra.decider.modules.DeciderModule import com.twitter.finatra.http.HttpServer import com.twitter.finatra.http.routing.HttpRouter import com.twitter.finatra.international.modules.I18nFactoryModule import com.twitter.finatra.international.modules.LanguagesModule import com.twitter.finatra.jackson.modules.ScalaObjectMapperModule import com.twitter.finatra.mtls.http.{Mtls => HttpMtls} import com.twitter.finatra.mtls.thriftmux.Mtls import com.twitter.finatra.thrift.ThriftServer import com.twitter.finatra.thrift.filters._ import com.twitter.finagle.thrift.Protocols import com.twitter.finatra.thrift.routing.ThriftRouter import com.twitter.follow_recommendations.common.clients.addressbook.AddressbookModule import com.twitter.follow_recommendations.common.clients.adserver.AdserverModule import com.twitter.follow_recommendations.common.clients.cache.MemcacheModule import com.twitter.follow_recommendations.common.clients.deepbirdv2.DeepBirdV2PredictionServiceClientModule import com.twitter.follow_recommendations.common.clients.email_storage_service.EmailStorageServiceModule import com.twitter.follow_recommendations.common.clients.geoduck.LocationServiceModule import com.twitter.follow_recommendations.common.clients.gizmoduck.GizmoduckModule import com.twitter.follow_recommendations.common.clients.graph_feature_service.GraphFeatureStoreModule import com.twitter.follow_recommendations.common.clients.impression_store.ImpressionStoreModule import com.twitter.follow_recommendations.common.clients.phone_storage_service.PhoneStorageServiceModule import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphModule import com.twitter.follow_recommendations.common.clients.strato.StratoClientModule import com.twitter.follow_recommendations.common.constants.ServiceConstants._ import com.twitter.follow_recommendations.common.feature_hydration.sources.HydrationSourcesModule import com.twitter.follow_recommendations.controllers.ThriftController import com.twitter.follow_recommendations.modules._ import com.twitter.follow_recommendations.service.exceptions.UnknownLoggingExceptionMapper import com.twitter.follow_recommendations.services.FollowRecommendationsServiceWarmupHandler import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService import com.twitter.geoduck.service.common.clientmodules.ReverseGeocoderThriftClientModule import com.twitter.inject.thrift.filters.DarkTrafficFilter import com.twitter.inject.thrift.modules.ThriftClientIdModule import com.twitter.product_mixer.core.controllers.ProductMixerController import com.twitter.product_mixer.core.module.PipelineExecutionLoggerModule import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule import com.twitter.product_mixer.core.product.guice.ProductScopeModule object FollowRecommendationsServiceThriftServerMain extends FollowRecommendationsServiceThriftServer class FollowRecommendationsServiceThriftServer extends ThriftServer with Mtls with HttpServer with HttpMtls { override val name: String = "follow-recommendations-service-server" override val modules: Seq[Module] = Seq( ABDeciderModule, AddressbookModule, AdserverModule, ConfigApiModule, DeciderModule, DeepBirdV2PredictionServiceClientModule, DiffyModule, EmailStorageServiceModule, FeaturesSwitchesModule, FlagsModule, GizmoduckModule, GraphFeatureStoreModule, HydrationSourcesModule, I18nFactoryModule, ImpressionStoreModule, LanguagesModule, LocationServiceModule, MemcacheModule, PhoneStorageServiceModule, PipelineExecutionLoggerModule, ProductMixerFlagModule, ProductRegistryModule, new ProductScopeModule(), new ProductScopeStringCenterModule(), new ReverseGeocoderThriftClientModule, ScalaObjectMapperModule, ScorerModule, ScribeModule, SocialGraphModule, StratoClientModule, ThriftClientIdModule, TimerModule, ) def configureThrift(router: ThriftRouter): Unit = { router .filter[LoggingMDCFilter] .filter[TraceIdMDCFilter] .filter[ThriftMDCFilter] .filter[StatsFilter] .filter[AccessLoggingFilter] .filter[ExceptionMappingFilter] .exceptionMapper[UnknownLoggingExceptionMapper] .filter[DarkTrafficFilter[FollowRecommendationsThriftService.ReqRepServicePerEndpoint]] .add[ThriftController] } override def configureThriftServer(server: ThriftMux.Server): ThriftMux.Server = { server.withProtocolFactory( Protocols.binaryFactory( stringLengthLimit = StringLengthLimit, containerLengthLimit = ContainerLengthLimit)) } override def configureHttp(router: HttpRouter): Unit = router.add( ProductMixerController[FollowRecommendationsThriftService.MethodPerEndpoint]( this.injector, FollowRecommendationsThriftService.ExecutePipeline)) override def warmup(): Unit = { handle[FollowRecommendationsServiceWarmupHandler]() } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Action.scala ================================================ package com.twitter.follow_recommendations.assembler.models import com.twitter.follow_recommendations.{thriftscala => t} case class Action(text: String, actionURL: String) { lazy val toThrift: t.Action = { t.Action(text, actionURL) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "stringcenter/client", ], exports = [ ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Config.scala ================================================ package com.twitter.follow_recommendations.assembler.models import com.twitter.stringcenter.client.core.ExternalString case class HeaderConfig(title: TitleConfig) case class TitleConfig(text: ExternalString) case class FooterConfig(actionConfig: Option[ActionConfig]) case class ActionConfig(footerText: ExternalString, actionURL: String) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/FeedbackAction.scala ================================================ package com.twitter.follow_recommendations.assembler.models import com.twitter.follow_recommendations.{thriftscala => t} trait FeedbackAction { def toThrift: t.FeedbackAction } case class DismissUserId() extends FeedbackAction { override lazy val toThrift: t.FeedbackAction = { t.FeedbackAction.DismissUserId(t.DismissUserId()) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Footer.scala ================================================ package com.twitter.follow_recommendations.assembler.models import com.twitter.follow_recommendations.{thriftscala => t} case class Footer(action: Option[Action]) { lazy val toThrift: t.Footer = { t.Footer(action.map(_.toThrift)) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Header.scala ================================================ package com.twitter.follow_recommendations.assembler.models import com.twitter.follow_recommendations.{thriftscala => t} case class Header(title: Title) { lazy val toThrift: t.Header = { t.Header(title.toThrift) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Layout.scala ================================================ package com.twitter.follow_recommendations.assembler.models sealed trait Layout case class UserListLayout( header: Option[HeaderConfig], userListOptions: UserListOptions, socialProofs: Option[Seq[SocialProof]], footer: Option[FooterConfig]) extends Layout case class CarouselLayout( header: Option[HeaderConfig], carouselOptions: CarouselOptions, socialProofs: Option[Seq[SocialProof]]) extends Layout ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/RecommendationOptions.scala ================================================ package com.twitter.follow_recommendations.assembler.models sealed trait RecommendationOptions case class UserListOptions( userBioEnabled: Boolean, userBioTruncated: Boolean, userBioMaxLines: Option[Long], ) extends RecommendationOptions case class CarouselOptions() extends RecommendationOptions ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/SocialProof.scala ================================================ package com.twitter.follow_recommendations.assembler.models import com.twitter.stringcenter.client.core.ExternalString sealed trait SocialProof case class GeoContextProof(popularInCountryText: ExternalString) extends SocialProof case class FollowedByUsersProof(text1: ExternalString, text2: ExternalString, textN: ExternalString) extends SocialProof sealed trait SocialText { def text: String } case class GeoSocialText(text: String) extends SocialText case class FollowedByUsersText(text: String) extends SocialText ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Title.scala ================================================ package com.twitter.follow_recommendations.assembler.models import com.twitter.follow_recommendations.{thriftscala => t} case class Title(text: String) { lazy val toThrift: t.Title = { t.Title(text) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/WTFPresentation.scala ================================================ package com.twitter.follow_recommendations.assembler.models import com.twitter.follow_recommendations.{thriftscala => t} trait WTFPresentation { def toThrift: t.WTFPresentation } case class UserList( userBioEnabled: Boolean, userBioTruncated: Boolean, userBioMaxLines: Option[Long], feedbackAction: Option[FeedbackAction]) extends WTFPresentation { def toThrift: t.WTFPresentation = { t.WTFPresentation.UserBioList( t.UserList(userBioEnabled, userBioTruncated, userBioMaxLines, feedbackAction.map(_.toThrift))) } } object UserList { def fromUserListOptions( userListOptions: UserListOptions ): UserList = { UserList( userListOptions.userBioEnabled, userListOptions.userBioTruncated, userListOptions.userBioMaxLines, None) } } case class Carousel( feedbackAction: Option[FeedbackAction]) extends WTFPresentation { def toThrift: t.WTFPresentation = { t.WTFPresentation.Carousel(t.Carousel(feedbackAction.map(_.toThrift))) } } object Carousel { def fromCarouselOptions( carouselOptions: CarouselOptions ): Carousel = { Carousel(None) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/PromotedAccountsBlender.scala ================================================ package com.twitter.follow_recommendations.blenders import com.google.common.annotations.VisibleForTesting import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.Transform import com.twitter.follow_recommendations.common.models.AdMetadata import com.twitter.follow_recommendations.common.models.Recommendation import com.twitter.inject.Logging import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class PromotedAccountsBlender @Inject() (statsReceiver: StatsReceiver) extends Transform[Int, Recommendation] with Logging { import PromotedAccountsBlender._ val stats = statsReceiver.scope(Name) val inputOrganicAccounts = stats.counter(InputOrganic) val inputPromotedAccounts = stats.counter(InputPromoted) val outputOrganicAccounts = stats.counter(OutputOrganic) val outputPromotedAccounts = stats.counter(OutputPromoted) val promotedAccountsStats = stats.scope(NumPromotedAccounts) override def transform( maxResults: Int, items: Seq[Recommendation] ): Stitch[Seq[Recommendation]] = { val (promoted, organic) = items.partition(_.isPromotedAccount) val promotedIds = promoted.map(_.id).toSet val dedupedOrganic = organic.filterNot(u => promotedIds.contains(u.id)) val blended = blendPromotedAccount(dedupedOrganic, promoted, maxResults) val (outputPromoted, outputOrganic) = blended.partition(_.isPromotedAccount) inputOrganicAccounts.incr(dedupedOrganic.size) inputPromotedAccounts.incr(promoted.size) outputOrganicAccounts.incr(outputOrganic.size) val size = outputPromoted.size outputPromotedAccounts.incr(size) if (size <= 5) { promotedAccountsStats.counter(outputPromoted.size.toString).incr() } else { promotedAccountsStats.counter(MoreThan5Promoted).incr() } Stitch.value(blended) } /** * Merge Promoted results and organic results. Promoted result dictates the position * in the merge list. * * merge a list of positioned users, aka. promoted, and a list of organic * users. The positioned promoted users are pre-sorted with regards to their * position ascendingly. Only requirement about position is to be within the * range, i.e, can not exceed the combined length if merge is successful, ok * to be at the last position, but not beyond. * For more detailed description of location position: * http://confluence.local.twitter.com/display/ADS/Promoted+Tweets+in+Timeline+Design+Document */ @VisibleForTesting private[blenders] def mergePromotedAccounts( organicUsers: Seq[Recommendation], promotedUsers: Seq[Recommendation] ): Seq[Recommendation] = { def mergeAccountWithIndex( organicUsers: Seq[Recommendation], promotedUsers: Seq[Recommendation], index: Int ): Stream[Recommendation] = { if (promotedUsers.isEmpty) organicUsers.toStream else { val promotedHead = promotedUsers.head val promotedTail = promotedUsers.tail promotedHead.adMetadata match { case Some(AdMetadata(position, _)) => if (position < 0) mergeAccountWithIndex(organicUsers, promotedTail, index) else if (position == index) promotedHead #:: mergeAccountWithIndex(organicUsers, promotedTail, index) else if (organicUsers.isEmpty) organicUsers.toStream else { val organicHead = organicUsers.head val organicTail = organicUsers.tail organicHead #:: mergeAccountWithIndex(organicTail, promotedUsers, index + 1) } case _ => logger.error("Unknown Candidate type in mergePromotedAccounts") Stream.empty } } } mergeAccountWithIndex(organicUsers, promotedUsers, 0) } private[this] def blendPromotedAccount( organic: Seq[Recommendation], promoted: Seq[Recommendation], maxResults: Int ): Seq[Recommendation] = { val merged = mergePromotedAccounts(organic, promoted) val mergedServed = merged.take(maxResults) val promotedServed = promoted.intersect(mergedServed) if (isBlendPromotedNeeded( mergedServed.size - promotedServed.size, promotedServed.size, maxResults )) { mergedServed } else { organic.take(maxResults) } } @VisibleForTesting private[blenders] def isBlendPromotedNeeded( organicSize: Int, promotedSize: Int, maxResults: Int ): Boolean = { (organicSize > 1) && (promotedSize > 0) && (promotedSize < organicSize) && (promotedSize <= 2) && (maxResults > 1) } } object PromotedAccountsBlender { val Name = "promoted_accounts_blender" val InputOrganic = "input_organic_accounts" val InputPromoted = "input_promoted_accounts" val OutputOrganic = "output_organic_accounts" val OutputPromoted = "output_promoted_accounts" val NumPromotedAccounts = "num_promoted_accounts" val MoreThan5Promoted = "more_than_5" } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "configapi/configapi-core", "configapi/configapi-decider", "configapi/configapi-featureswitches:v2", "featureswitches/featureswitches-core", "featureswitches/featureswitches-core:v2", "featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ConfigBuilder.scala ================================================ package com.twitter.follow_recommendations.configapi import com.twitter.timelines.configapi.CompositeConfig import com.twitter.timelines.configapi.Config import javax.inject.Inject import javax.inject.Singleton @Singleton class ConfigBuilder @Inject() ( deciderConfigs: DeciderConfigs, featureSwitchConfigs: FeatureSwitchConfigs) { // The order of configs added to `CompositeConfig` is important. The config will be matched with // the first possible rule. So, current setup will give priority to Deciders instead of FS def build(): Config = new CompositeConfig(Seq(deciderConfigs.config, featureSwitchConfigs.config)) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/DeciderConfigs.scala ================================================ package com.twitter.follow_recommendations.configapi import com.twitter.decider.Recipient import com.twitter.decider.SimpleRecipient import com.twitter.follow_recommendations.configapi.deciders.DeciderKey import com.twitter.follow_recommendations.configapi.deciders.DeciderParams import com.twitter.follow_recommendations.products.home_timeline_tweet_recs.configapi.HomeTimelineTweetRecsParams import com.twitter.servo.decider.DeciderGateBuilder import com.twitter.timelines.configapi._ import com.twitter.timelines.configapi.decider.DeciderSwitchOverrideValue import com.twitter.timelines.configapi.decider.GuestRecipient import com.twitter.timelines.configapi.decider.RecipientBuilder import javax.inject.Inject import javax.inject.Singleton @Singleton class DeciderConfigs @Inject() (deciderGateBuilder: DeciderGateBuilder) { val overrides: Seq[OptionalOverride[_]] = DeciderConfigs.ParamsToDeciderMap.map { case (params, deciderKey) => params.optionalOverrideValue( DeciderSwitchOverrideValue( feature = deciderGateBuilder.keyToFeature(deciderKey), enabledValue = true, recipientBuilder = DeciderConfigs.UserOrGuestOrRequest ) ) }.toSeq val config: BaseConfig = BaseConfigBuilder(overrides).build("FollowRecommendationServiceDeciders") } object DeciderConfigs { val ParamsToDeciderMap = Map( DeciderParams.EnableRecommendations -> DeciderKey.EnableRecommendations, DeciderParams.EnableScoreUserCandidates -> DeciderKey.EnableScoreUserCandidates, HomeTimelineTweetRecsParams.EnableProduct -> DeciderKey.EnableHomeTimelineTweetRecsProduct, ) object UserOrGuestOrRequest extends RecipientBuilder { def apply(requestContext: BaseRequestContext): Option[Recipient] = requestContext match { case c: WithUserId if c.userId.isDefined => c.userId.map(SimpleRecipient) case c: WithGuestId if c.guestId.isDefined => c.guestId.map(GuestRecipient) case c: WithGuestId => RecipientBuilder.Request(c) case _ => throw new UndefinedUserIdNorGuestIDException(requestContext) } } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/FeatureSwitchConfigs.scala ================================================ package com.twitter.follow_recommendations.configapi import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.candidate_sources.base.SocialProofEnforcedCandidateSourceFSConfig import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsFSConfig import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeoQualityFollowSourceFSConfig import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsFSConfig import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeoSourceFSConfig import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSourceFSConfig import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonFSConfig import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsFSConfig import com.twitter.follow_recommendations.common.candidate_sources.sims.SimsSourceFSConfig import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersFSConfig import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.SimsExpansionFSConfig import com.twitter.follow_recommendations.common.candidate_sources.socialgraph.RecentFollowingRecentFollowingExpansionSourceFSConfig import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStpSourceFsConfig import com.twitter.follow_recommendations.common.candidate_sources.stp.OnlineSTPSourceFSConfig import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsFSConfig import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphFSConfig import com.twitter.follow_recommendations.common.feature_hydration.sources.FeatureHydrationSourcesFSConfig import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.WeightedCandidateSourceRankerFSConfig import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.follow_recommendations.flows.content_recommender_flow.ContentRecommenderFlowFSConfig import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicateFSConfig import com.twitter.follow_recommendations.common.predicates.hss.HssPredicateFSConfig import com.twitter.follow_recommendations.common.predicates.sgs.SgsPredicateFSConfig import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlFlowFSConfig import com.twitter.logging.Logger import com.twitter.timelines.configapi.BaseConfigBuilder import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil import javax.inject.Inject import javax.inject.Singleton @Singleton class FeatureSwitchConfigs @Inject() ( globalFeatureSwitchConfig: GlobalFeatureSwitchConfig, featureHydrationSourcesFSConfig: FeatureHydrationSourcesFSConfig, weightedCandidateSourceRankerFSConfig: WeightedCandidateSourceRankerFSConfig, // Flow related config contentRecommenderFlowFSConfig: ContentRecommenderFlowFSConfig, postNuxMlFlowFSConfig: PostNuxMlFlowFSConfig, // Candidate source related config crowdSearchAccountsFSConfig: CrowdSearchAccountsFSConfig, offlineStpSourceFsConfig: OfflineStpSourceFsConfig, onlineSTPSourceFSConfig: OnlineSTPSourceFSConfig, popGeoSourceFSConfig: PopGeoSourceFSConfig, popGeoQualityFollowFSConfig: PopGeoQualityFollowSourceFSConfig, realGraphOonFSConfig: RealGraphOonFSConfig, repeatedProfileVisitsFSConfig: RepeatedProfileVisitsFSConfig, recentEngagementSimilarUsersFSConfig: RecentEngagementSimilarUsersFSConfig, recentFollowingRecentFollowingExpansionSourceFSConfig: RecentFollowingRecentFollowingExpansionSourceFSConfig, simsExpansionFSConfig: SimsExpansionFSConfig, simsSourceFSConfig: SimsSourceFSConfig, socialProofEnforcedCandidateSourceFSConfig: SocialProofEnforcedCandidateSourceFSConfig, triangularLoopsFSConfig: TriangularLoopsFSConfig, userUserGraphFSConfig: UserUserGraphFSConfig, // Predicate related configs gizmoduckPredicateFSConfig: GizmoduckPredicateFSConfig, hssPredicateFSConfig: HssPredicateFSConfig, sgsPredicateFSConfig: SgsPredicateFSConfig, ppmiLocaleSourceFSConfig: PPMILocaleFollowSourceFSConfig, topOrganicFollowsAccountsFSConfig: TopOrganicFollowsAccountsFSConfig, statsReceiver: StatsReceiver) { val logger = Logger(classOf[FeatureSwitchConfigs]) val mergedFSConfig = FeatureSwitchConfig.merge( Seq( globalFeatureSwitchConfig, featureHydrationSourcesFSConfig, weightedCandidateSourceRankerFSConfig, // Flow related config contentRecommenderFlowFSConfig, postNuxMlFlowFSConfig, // Candidate source related config crowdSearchAccountsFSConfig, offlineStpSourceFsConfig, onlineSTPSourceFSConfig, popGeoSourceFSConfig, popGeoQualityFollowFSConfig, realGraphOonFSConfig, repeatedProfileVisitsFSConfig, recentEngagementSimilarUsersFSConfig, recentFollowingRecentFollowingExpansionSourceFSConfig, simsExpansionFSConfig, simsSourceFSConfig, socialProofEnforcedCandidateSourceFSConfig, triangularLoopsFSConfig, userUserGraphFSConfig, // Predicate related configs: gizmoduckPredicateFSConfig, hssPredicateFSConfig, sgsPredicateFSConfig, ppmiLocaleSourceFSConfig, topOrganicFollowsAccountsFSConfig, ) ) /** * enum params have to be listed in this main file together as otherwise we'll have to pass in * some signature like `Seq[FSEnumParams[_]]` which are generics of generics and won't compile. * we only have enumFsParams from globalFeatureSwitchConfig at the moment */ val enumOverrides = globalFeatureSwitchConfig.enumFsParams.flatMap { enumParam => FeatureSwitchOverrideUtil.getEnumFSOverrides(statsReceiver, logger, enumParam) } val gatedOverrides = mergedFSConfig.gatedOverridesMap.flatMap { case (fsName, overrides) => FeatureSwitchOverrideUtil.gatedOverrides(fsName, overrides: _*) } val enumSeqOverrides = globalFeatureSwitchConfig.enumSeqFsParams.flatMap { enumSeqParam => FeatureSwitchOverrideUtil.getEnumSeqFSOverrides(statsReceiver, logger, enumSeqParam) } val overrides = FeatureSwitchOverrideUtil .getBooleanFSOverrides(mergedFSConfig.booleanFSParams: _*) ++ FeatureSwitchOverrideUtil .getBoundedIntFSOverrides(mergedFSConfig.intFSParams: _*) ++ FeatureSwitchOverrideUtil .getBoundedLongFSOverrides(mergedFSConfig.longFSParams: _*) ++ FeatureSwitchOverrideUtil .getBoundedDoubleFSOverrides(mergedFSConfig.doubleFSParams: _*) ++ FeatureSwitchOverrideUtil .getDurationFSOverrides(mergedFSConfig.durationFSParams: _*) ++ FeatureSwitchOverrideUtil .getBoundedOptionalDoubleOverrides(mergedFSConfig.optionalDoubleFSParams: _*) ++ FeatureSwitchOverrideUtil.getStringSeqFSOverrides(mergedFSConfig.stringSeqFSParams: _*) ++ enumOverrides ++ gatedOverrides ++ enumSeqOverrides val config = BaseConfigBuilder(overrides).build("FollowRecommendationServiceFeatureSwitches") } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/GlobalFeatureSwitchConfig.scala ================================================ package com.twitter.follow_recommendations.configapi import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsParams.AccountsFilteringAndRankingLogics import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsParams.{ AccountsFilteringAndRankingLogics => OrganicAccountsFilteringAndRankingLogics } import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersParams import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.SimsExpansionSourceParams import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.MlRankerParams.CandidateScorerIdParam import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.follow_recommendations.configapi.params.GlobalParams.CandidateSourcesToFilter import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableCandidateParamHydrations import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableGFSSocialProofTransform import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableRecommendationFlowLogs import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableWhoToFollowProducts import com.twitter.follow_recommendations.configapi.params.GlobalParams.KeepSocialUserCandidate import com.twitter.follow_recommendations.configapi.params.GlobalParams.KeepUserCandidate import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.Param import javax.inject.Inject import javax.inject.Singleton @Singleton class GlobalFeatureSwitchConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[Param[Boolean] with FSName] = { Seq( EnableCandidateParamHydrations, KeepUserCandidate, KeepSocialUserCandidate, EnableGFSSocialProofTransform, EnableWhoToFollowProducts, EnableRecommendationFlowLogs ) } val enumFsParams = Seq( CandidateScorerIdParam, SimsExpansionSourceParams.Aggregator, RecentEngagementSimilarUsersParams.Aggregator, CandidateSourcesToFilter, ) val enumSeqFsParams = Seq( AccountsFilteringAndRankingLogics, OrganicAccountsFilteringAndRankingLogics ) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ParamsFactory.scala ================================================ package com.twitter.follow_recommendations.configapi import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.product_mixer.core.model.marshalling.request.ClientContext import com.twitter.servo.util.MemoizingStatsReceiver import com.twitter.timelines.configapi.Config import com.twitter.timelines.configapi.FeatureValue import com.twitter.timelines.configapi.Params import javax.inject.Inject import javax.inject.Singleton @Singleton class ParamsFactory @Inject() ( config: Config, requestContextFactory: RequestContextFactory, statsReceiver: StatsReceiver) { private val stats = new MemoizingStatsReceiver(statsReceiver.scope("configapi")) def apply(followRecommendationServiceRequestContext: RequestContext): Params = config(followRecommendationServiceRequestContext, stats) def apply( clientContext: ClientContext, displayLocation: DisplayLocation, featureOverrides: Map[String, FeatureValue] ): Params = apply(requestContextFactory(clientContext, displayLocation, featureOverrides)) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContext.scala ================================================ package com.twitter.follow_recommendations.configapi import com.twitter.timelines.configapi.BaseRequestContext import com.twitter.timelines.configapi.FeatureContext import com.twitter.timelines.configapi.NullFeatureContext import com.twitter.timelines.configapi.GuestId import com.twitter.timelines.configapi.UserId import com.twitter.timelines.configapi.WithFeatureContext import com.twitter.timelines.configapi.WithGuestId import com.twitter.timelines.configapi.WithUserId case class RequestContext( userId: Option[UserId], guestId: Option[GuestId], featureContext: FeatureContext = NullFeatureContext) extends BaseRequestContext with WithUserId with WithGuestId with WithFeatureContext ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContextFactory.scala ================================================ package com.twitter.follow_recommendations.configapi import com.google.common.annotations.VisibleForTesting import com.google.inject.Inject import com.twitter.decider.Decider import com.twitter.featureswitches.v2.FeatureSwitches import com.twitter.featureswitches.{Recipient => FeatureSwitchRecipient} import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.product_mixer.core.model.marshalling.request.ClientContext import com.twitter.snowflake.id.SnowflakeId import com.twitter.timelines.configapi.FeatureContext import com.twitter.timelines.configapi.FeatureValue import com.twitter.timelines.configapi.ForcedFeatureContext import com.twitter.timelines.configapi.OrElseFeatureContext import com.twitter.timelines.configapi.featureswitches.v2.FeatureSwitchResultsFeatureContext import javax.inject.Singleton /* * Request Context Factory is used to build RequestContext objects which are used * by the config api to determine the param overrides to apply to the request. * The param overrides are determined per request by configs which specify which * FS/Deciders/AB translate to what param overrides. */ @Singleton class RequestContextFactory @Inject() (featureSwitches: FeatureSwitches, decider: Decider) { def apply( clientContext: ClientContext, displayLocation: DisplayLocation, featureOverrides: Map[String, FeatureValue] ): RequestContext = { val featureContext = getFeatureContext(clientContext, displayLocation, featureOverrides) RequestContext(clientContext.userId, clientContext.guestId, featureContext) } private[configapi] def getFeatureContext( clientContext: ClientContext, displayLocation: DisplayLocation, featureOverrides: Map[String, FeatureValue] ): FeatureContext = { val recipient = getFeatureSwitchRecipient(clientContext) .withCustomFields("display_location" -> displayLocation.toFsName) // userAgeOpt is going to be set to None for logged out users and defaulted to Some(Int.MaxValue) for non-snowflake users val userAgeOpt = clientContext.userId.map { userId => SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue) } val recipientWithAccountAge = userAgeOpt .map(age => recipient.withCustomFields("account_age_in_days" -> age)).getOrElse(recipient) val results = featureSwitches.matchRecipient(recipientWithAccountAge) OrElseFeatureContext( ForcedFeatureContext(featureOverrides), new FeatureSwitchResultsFeatureContext(results)) } @VisibleForTesting private[configapi] def getFeatureSwitchRecipient( clientContext: ClientContext ): FeatureSwitchRecipient = { FeatureSwitchRecipient( userId = clientContext.userId, userRoles = clientContext.userRoles, deviceId = clientContext.deviceId, guestId = clientContext.guestId, languageCode = clientContext.languageCode, countryCode = clientContext.countryCode, isVerified = None, clientApplicationId = clientContext.appId, isTwoffice = clientContext.isTwoffice ) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "configapi/configapi-core", "configapi/configapi-decider", "configapi/configapi-featureswitches:v2", "featureswitches/featureswitches-core", "featureswitches/featureswitches-core:v2", "featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContext.scala ================================================ package com.twitter.follow_recommendations.configapi.candidates import com.twitter.timelines.configapi.BaseRequestContext import com.twitter.timelines.configapi.FeatureContext import com.twitter.timelines.configapi.NullFeatureContext import com.twitter.timelines.configapi.WithFeatureContext import com.twitter.timelines.configapi.WithUserId /** * represent the context for a recommendation candidate (producer side) * @param userId id of the recommended user * @param featureContext feature context */ case class CandidateUserContext( override val userId: Option[Long], featureContext: FeatureContext = NullFeatureContext) extends BaseRequestContext with WithUserId with WithFeatureContext ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContextFactory.scala ================================================ package com.twitter.follow_recommendations.configapi.candidates import com.google.common.annotations.VisibleForTesting import com.google.inject.Inject import com.twitter.decider.Decider import com.twitter.featureswitches.v2.FeatureSwitches import com.twitter.featureswitches.{Recipient => FeatureSwitchRecipient} import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants.PRODUCER_SIDE_FEATURE_SWITCHES import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.timelines.configapi.FeatureContext import com.twitter.timelines.configapi.featureswitches.v2.FeatureSwitchResultsFeatureContext import javax.inject.Named import javax.inject.Singleton @Singleton class CandidateUserContextFactory @Inject() ( @Named(PRODUCER_SIDE_FEATURE_SWITCHES) featureSwitches: FeatureSwitches, decider: Decider) { def apply( candidateUser: CandidateUser, displayLocation: DisplayLocation ): CandidateUserContext = { val featureContext = getFeatureContext(candidateUser, displayLocation) CandidateUserContext(Some(candidateUser.id), featureContext) } private[configapi] def getFeatureContext( candidateUser: CandidateUser, displayLocation: DisplayLocation ): FeatureContext = { val recipient = getFeatureSwitchRecipient(candidateUser).withCustomFields( "display_location" -> displayLocation.toFsName) new FeatureSwitchResultsFeatureContext(featureSwitches.matchRecipient(recipient)) } @VisibleForTesting private[configapi] def getFeatureSwitchRecipient( candidateUser: CandidateUser ): FeatureSwitchRecipient = { FeatureSwitchRecipient( userId = Some(candidateUser.id), userRoles = None, deviceId = None, guestId = None, languageCode = None, countryCode = None, isVerified = None, clientApplicationId = None, isTwoffice = None ) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserParamsFactory.scala ================================================ package com.twitter.follow_recommendations.configapi.candidates import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.configapi.params.GlobalParams import com.twitter.servo.util.MemoizingStatsReceiver import com.twitter.timelines.configapi.Config import com.twitter.timelines.configapi.HasParams import com.twitter.timelines.configapi.Params import javax.inject.Inject import javax.inject.Singleton /** * CandidateParamsFactory is primarily used for "producer side" experiments, don't use it on consumer side experiments */ @Singleton class CandidateUserParamsFactory[T <: HasParams with HasDisplayLocation] @Inject() ( config: Config, candidateContextFactory: CandidateUserContextFactory, statsReceiver: StatsReceiver) { private val stats = new MemoizingStatsReceiver(statsReceiver.scope("configapi_candidate_params")) def apply(candidateContext: CandidateUser, request: T): CandidateUser = { if (candidateContext.params == Params.Invalid) { if (request.params(GlobalParams.EnableCandidateParamHydrations)) { candidateContext.copy(params = config(candidateContextFactory(candidateContext, request.displayLocation), stats)) } else { candidateContext.copy(params = Params.Empty) } } else { candidateContext } } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/HydrateCandidateParamsTransform.scala ================================================ package com.twitter.follow_recommendations.configapi.candidates import com.google.inject.Inject import com.google.inject.Singleton import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.common.base.Transform import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.HasParams import com.twitter.util.logging.Logging @Singleton class HydrateCandidateParamsTransform[Target <: HasParams with HasDisplayLocation] @Inject() ( candidateParamsFactory: CandidateUserParamsFactory[Target]) extends Transform[Target, CandidateUser] with Logging { def transform(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] = { Stitch.value(candidates.map(candidateParamsFactory.apply(_, target))) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "configapi/configapi-core", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/FeatureSwitchConfig.scala ================================================ package com.twitter.follow_recommendations.configapi.common import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil.DefinedFeatureName import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil.ValueFeatureName import com.twitter.timelines.configapi.BoundedParam import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.timelines.configapi.OptionalOverride import com.twitter.timelines.configapi.Param import com.twitter.util.Duration trait FeatureSwitchConfig { def booleanFSParams: Seq[Param[Boolean] with FSName] = Nil def intFSParams: Seq[FSBoundedParam[Int]] = Nil def longFSParams: Seq[FSBoundedParam[Long]] = Nil def doubleFSParams: Seq[FSBoundedParam[Double]] = Nil def durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Nil def optionalDoubleFSParams: Seq[ (BoundedParam[Option[Double]], DefinedFeatureName, ValueFeatureName) ] = Nil def stringSeqFSParams: Seq[Param[Seq[String]] with FSName] = Nil /** * Apply overrides in list when the given FS Key is enabled. * This override type does NOT work with experiments. Params here will be evaluated for every * request IMMEDIATELY, not upon param.apply. If you would like to use an experiment pls use * the primitive type or ENUM overrides. */ def gatedOverridesMap: Map[String, Seq[OptionalOverride[_]]] = Map.empty } object FeatureSwitchConfig { def merge(configs: Seq[FeatureSwitchConfig]): FeatureSwitchConfig = new FeatureSwitchConfig { override def booleanFSParams: Seq[Param[Boolean] with FSName] = configs.flatMap(_.booleanFSParams) override def intFSParams: Seq[FSBoundedParam[Int]] = configs.flatMap(_.intFSParams) override def longFSParams: Seq[FSBoundedParam[Long]] = configs.flatMap(_.longFSParams) override def durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = configs.flatMap(_.durationFSParams) override def gatedOverridesMap: Map[String, Seq[OptionalOverride[_]]] = configs.flatMap(_.gatedOverridesMap).toMap override def doubleFSParams: Seq[FSBoundedParam[Double]] = configs.flatMap(_.doubleFSParams) override def optionalDoubleFSParams: Seq[ (BoundedParam[Option[Double]], DefinedFeatureName, ValueFeatureName) ] = configs.flatMap(_.optionalDoubleFSParams) override def stringSeqFSParams: Seq[Param[Seq[String]] with FSName] = configs.flatMap(_.stringSeqFSParams) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "configapi/configapi-core", "configapi/configapi-decider", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderKey.scala ================================================ package com.twitter.follow_recommendations.configapi.deciders import com.twitter.servo.decider.DeciderKeyEnum object DeciderKey extends DeciderKeyEnum { val EnableDiffyModuleDarkReading = Value("enable_diffy_module_dark_reading") val EnableRecommendations = Value("enable_recommendations") val EnableScoreUserCandidates = Value("enable_score_user_candidates") val EnableProfileSidebarProduct = Value("enable_profile_sidebar_product") val EnableMagicRecsProduct = Value("enable_magic_recs_product") val EnableRuxLandingPageProduct = Value("enable_rux_landing_page_product") val EnableRuxPymkProduct = Value("enable_rux_pymk_product") val EnableProfileBonusFollowProduct = Value("enable_profile_bonus_follow_product") val EnableElectionExploreWtfProduct = Value("enable_election_explore_wtf_product") val EnableClusterFollowProduct = Value("enable_cluster_follow_product") val EnableHomeTimelineProduct = Value("enable_home_timeline_product") val EnableHtlBonusFollowProduct = Value("enable_htl_bonus_follow_product") val EnableExploreTabProduct = Value("enable_explore_tab_product") val EnableSidebarProduct = Value("enable_sidebar_product") val EnableNuxPymkProduct = Value("enable_nux_pymk_product") val EnableNuxInterestsProduct = Value("enable_nux_interests_product") val EnableNuxTopicBonusFollowProduct = Value("enable_nux_topic_bonus_follow_product") val EnableCampaignFormProduct = Value("enable_campaign_form_product") val EnableReactiveFollowProduct = Value("enable_reactive_follow_product") val EnableIndiaCovid19CuratedAccountsWtfProduct = Value( "enable_india_covid19_curated_accounts_wtf_product") val EnableAbUploadProduct = Value("enable_ab_upload_product") val EnablePeolePlusPlusProduct = Value("enable_people_plus_plus_product") val EnableTweetNotificationRecsProduct = Value("enable_tweet_notification_recs_product") val EnableProfileDeviceFollow = Value("enable_profile_device_follow_product") val EnableRecosBackfillProduct = Value("enable_recos_backfill_product") val EnablePostNuxFollowTaskProduct = Value("enable_post_nux_follow_task_product") val EnableCuratedSpaceHostsProduct = Value("enable_curated_space_hosts_product") val EnableNuxGeoCategoryProduct = Value("enable_nux_geo_category_product") val EnableNuxInterestsCategoryProduct = Value("enable_nux_interests_category_product") val EnableNuxPymkCategoryProduct = Value("enable_nux_pymk_category_product") val EnableHomeTimelineTweetRecsProduct = Value("enable_home_timeline_tweet_recs_product") val EnableHtlBulkFriendFollowsProduct = Value("enable_htl_bulk_friend_follows_product") val EnableNuxAutoFollowProduct = Value("enable_nux_auto_follow_product") val EnableSearchBonusFollowProduct = Value("enable_search_bonus_follow_product") val EnableFetchUserInRequestBuilder = Value("enable_fetch_user_in_request_builder") val EnableProductMixerMagicRecsProduct = Value("enable_product_mixer_magic_recs_product") val EnableHomeTimelineReverseChronProduct = Value("enable_home_timeline_reverse_chron_product") val EnableProductMixerPipelineMagicRecsDarkRead = Value( "enable_product_mixer_pipeline_magic_recs_dark_read") val EnableExperimentalCaching = Value("enable_experimental_caching") val EnableDistributedCaching = Value("enable_distributed_caching") val EnableGizmoduckCaching = Value("enable_gizmoduck_caching") val EnableTrafficDarkReading = Value("enable_traffic_dark_reading") val EnableGraphFeatureServiceRequests = Value("enable_graph_feature_service_requests") } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderParams.scala ================================================ package com.twitter.follow_recommendations.configapi.deciders import com.twitter.timelines.configapi.Param object DeciderParams { object EnableRecommendations extends Param[Boolean](false) object EnableScoreUserCandidates extends Param[Boolean](false) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "configapi/configapi-core", "configapi/configapi-decider", "configapi/configapi-featureswitches:v2", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/GlobalParams.scala ================================================ package com.twitter.follow_recommendations.configapi.params import com.twitter.follow_recommendations.models.CandidateSourceType import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSParam /** * When adding Producer side experiments, make sure to register the FS Key in [[ProducerFeatureFilter]] * in [[FeatureSwitchesModule]], otherwise, the FS will not work. */ object GlobalParams { object EnableCandidateParamHydrations extends FSParam[Boolean]("frs_receiver_enable_candidate_params", false) object KeepUserCandidate extends FSParam[Boolean]("frs_receiver_holdback_keep_user_candidate", true) object KeepSocialUserCandidate extends FSParam[Boolean]("frs_receiver_holdback_keep_social_user_candidate", true) case object EnableGFSSocialProofTransform extends FSParam("social_proof_transform_use_graph_feature_service", true) case object EnableWhoToFollowProducts extends FSParam("who_to_follow_product_enabled", true) case object CandidateSourcesToFilter extends FSEnumParam[CandidateSourceType.type]( "candidate_sources_type_filter_id", CandidateSourceType.None, CandidateSourceType) object EnableRecommendationFlowLogs extends FSParam[Boolean]("frs_recommendation_flow_logs_enabled", false) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/javax/inject:javax.inject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "decider/src/main/scala", "finagle/finagle-core/src/main", "finatra/inject/inject-core/src/main/scala", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift:controller", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/exceptions", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/filters", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/modules", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/response", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/routing", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services", "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query", "scrooge/scrooge-core/src/main/scala", "util/util-core:scala", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/CandidateUserDebugParamsBuilder.scala ================================================ package com.twitter.follow_recommendations.controllers import com.twitter.follow_recommendations.common.models._ import com.twitter.follow_recommendations.configapi.ParamsFactory import com.twitter.follow_recommendations.models.CandidateUserDebugParams import com.twitter.follow_recommendations.models.FeatureValue import com.twitter.follow_recommendations.{thriftscala => t} import javax.inject.Inject import javax.inject.Singleton @Singleton class CandidateUserDebugParamsBuilder @Inject() (paramsFactory: ParamsFactory) { def fromThrift(req: t.ScoringUserRequest): CandidateUserDebugParams = { val clientContext = ClientContextConverter.fromThrift(req.clientContext) val displayLocation = DisplayLocation.fromThrift(req.displayLocation) CandidateUserDebugParams(req.candidates.map { candidate => candidate.userId -> paramsFactory( clientContext, displayLocation, candidate.featureOverrides .map(_.mapValues(FeatureValue.fromThrift).toMap).getOrElse(Map.empty)) }.toMap) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RecommendationRequestBuilder.scala ================================================ package com.twitter.follow_recommendations.controllers import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.models.ClientContextConverter import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.follow_recommendations.models.DebugParams import com.twitter.follow_recommendations.models.DisplayContext import com.twitter.follow_recommendations.models.RecommendationRequest import com.twitter.follow_recommendations.{thriftscala => t} import com.twitter.gizmoduck.thriftscala.UserType import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class RecommendationRequestBuilder @Inject() ( requestBuilderUserFetcher: RequestBuilderUserFetcher, statsReceiver: StatsReceiver) { private val scopedStats = statsReceiver.scope(this.getClass.getSimpleName) private val isSoftUserCounter = scopedStats.counter("is_soft_user") def fromThrift(tRequest: t.RecommendationRequest): Stitch[RecommendationRequest] = { requestBuilderUserFetcher.fetchUser(tRequest.clientContext.userId).map { userOpt => val isSoftUser = userOpt.exists(_.userType == UserType.Soft) if (isSoftUser) isSoftUserCounter.incr() RecommendationRequest( clientContext = ClientContextConverter.fromThrift(tRequest.clientContext), displayLocation = DisplayLocation.fromThrift(tRequest.displayLocation), displayContext = tRequest.displayContext.map(DisplayContext.fromThrift), maxResults = tRequest.maxResults, cursor = tRequest.cursor, excludedIds = tRequest.excludedIds, fetchPromotedContent = tRequest.fetchPromotedContent, debugParams = tRequest.debugParams.map(DebugParams.fromThrift), userLocationState = tRequest.userLocationState, isSoftUser = isSoftUser ) } } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RequestBuilderUserFetcher.scala ================================================ package com.twitter.follow_recommendations.controllers import com.twitter.decider.Decider import com.twitter.decider.SimpleRecipient import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.StatsUtil import com.twitter.follow_recommendations.configapi.deciders.DeciderKey import com.twitter.gizmoduck.thriftscala.LookupContext import com.twitter.gizmoduck.thriftscala.User import com.twitter.stitch.Stitch import com.twitter.stitch.gizmoduck.Gizmoduck import javax.inject.Inject import javax.inject.Singleton @Singleton class RequestBuilderUserFetcher @Inject() ( gizmoduck: Gizmoduck, statsReceiver: StatsReceiver, decider: Decider) { private val scopedStats = statsReceiver.scope(this.getClass.getSimpleName) def fetchUser(userIdOpt: Option[Long]): Stitch[Option[User]] = { userIdOpt match { case Some(userId) if enableDecider(userId) => val stitch = gizmoduck .getUserById( userId = userId, context = LookupContext( forUserId = Some(userId), includeProtected = true, includeSoftUsers = true ) ).map(user => Some(user)) StatsUtil .profileStitch(stitch, scopedStats) .handle { case _: Throwable => None } case _ => Stitch.None } } private def enableDecider(userId: Long): Boolean = { decider.isAvailable( DeciderKey.EnableFetchUserInRequestBuilder.toString, Some(SimpleRecipient(userId))) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ScoringUserRequestBuilder.scala ================================================ package com.twitter.follow_recommendations.controllers import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.ClientContextConverter import com.twitter.follow_recommendations.common.models.DebugOptions import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.follow_recommendations.models.DebugParams import com.twitter.follow_recommendations.models.ScoringUserRequest import com.twitter.timelines.configapi.Params import javax.inject.Inject import javax.inject.Singleton import com.twitter.follow_recommendations.{thriftscala => t} import com.twitter.gizmoduck.thriftscala.UserType import com.twitter.stitch.Stitch @Singleton class ScoringUserRequestBuilder @Inject() ( requestBuilderUserFetcher: RequestBuilderUserFetcher, candidateUserDebugParamsBuilder: CandidateUserDebugParamsBuilder, statsReceiver: StatsReceiver) { private val scopedStats = statsReceiver.scope(this.getClass.getSimpleName) private val isSoftUserCounter = scopedStats.counter("is_soft_user") def fromThrift(req: t.ScoringUserRequest): Stitch[ScoringUserRequest] = { requestBuilderUserFetcher.fetchUser(req.clientContext.userId).map { userOpt => val isSoftUser = userOpt.exists(_.userType == UserType.Soft) if (isSoftUser) isSoftUserCounter.incr() val candidateUsersParamsMap = candidateUserDebugParamsBuilder.fromThrift(req) val candidates = req.candidates.map { candidate => CandidateUser .fromUserRecommendation(candidate).copy(params = candidateUsersParamsMap.paramsMap.getOrElse(candidate.userId, Params.Invalid)) } ScoringUserRequest( clientContext = ClientContextConverter.fromThrift(req.clientContext), displayLocation = DisplayLocation.fromThrift(req.displayLocation), params = Params.Empty, debugOptions = req.debugParams.map(DebugOptions.fromDebugParamsThrift), recentFollowedUserIds = None, recentFollowedByUserIds = None, wtfImpressions = None, similarToUserIds = Nil, candidates = candidates, debugParams = req.debugParams.map(DebugParams.fromThrift), isSoftUser = isSoftUser ) } } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ThriftController.scala ================================================ package com.twitter.follow_recommendations.controllers import com.twitter.finatra.thrift.Controller import com.twitter.follow_recommendations.configapi.ParamsFactory import com.twitter.follow_recommendations.services.ProductPipelineSelector import com.twitter.follow_recommendations.services.UserScoringService import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService._ import com.twitter.stitch.Stitch import javax.inject.Inject class ThriftController @Inject() ( userScoringService: UserScoringService, recommendationRequestBuilder: RecommendationRequestBuilder, scoringUserRequestBuilder: ScoringUserRequestBuilder, productPipelineSelector: ProductPipelineSelector, paramsFactory: ParamsFactory) extends Controller(FollowRecommendationsThriftService) { handle(GetRecommendations) { args: GetRecommendations.Args => val stitch = recommendationRequestBuilder.fromThrift(args.request).flatMap { request => val params = paramsFactory( request.clientContext, request.displayLocation, request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty)) productPipelineSelector.selectPipeline(request, params).map(_.toThrift) } Stitch.run(stitch) } handle(ScoreUserCandidates) { args: ScoreUserCandidates.Args => val stitch = scoringUserRequestBuilder.fromThrift(args.request).flatMap { request => val params = paramsFactory( request.clientContext, request.displayLocation, request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty)) userScoringService.get(request.copy(params = params)).map(_.toThrift) } Stitch.run(stitch) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/promoted_accounts", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlow.scala ================================================ package com.twitter.follow_recommendations.flows.ads import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource import com.twitter.follow_recommendations.common.base.IdentityRanker import com.twitter.follow_recommendations.common.base.IdentityTransform import com.twitter.follow_recommendations.common.base.ParamPredicate import com.twitter.follow_recommendations.common.base.Predicate import com.twitter.follow_recommendations.common.base.Ranker import com.twitter.follow_recommendations.common.base.RecommendationFlow import com.twitter.follow_recommendations.common.base.RecommendationResultsConfig import com.twitter.follow_recommendations.common.base.Transform import com.twitter.follow_recommendations.common.base.TruePredicate import com.twitter.follow_recommendations.common.candidate_sources.promoted_accounts.PromotedAccountsCandidateSource import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.predicates.ExcludedUserIdPredicate import com.twitter.follow_recommendations.common.transforms.tracking_token.TrackingTokenTransform import com.twitter.inject.annotations.Flag import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.util.Duration import javax.inject.Inject import javax.inject.Singleton @Singleton class PromotedAccountsFlow @Inject() ( promotedAccountsCandidateSource: PromotedAccountsCandidateSource, trackingTokenTransform: TrackingTokenTransform, baseStatsReceiver: StatsReceiver, @Flag("fetch_prod_promoted_accounts") fetchProductionPromotedAccounts: Boolean) extends RecommendationFlow[PromotedAccountsFlowRequest, CandidateUser] { protected override def targetEligibility: Predicate[PromotedAccountsFlowRequest] = new ParamPredicate[PromotedAccountsFlowRequest]( PromotedAccountsFlowParams.TargetEligibility ) protected override def candidateSources( target: PromotedAccountsFlowRequest ): Seq[CandidateSource[PromotedAccountsFlowRequest, CandidateUser]] = { import EnrichedCandidateSource._ val candidateSourceStats = statsReceiver.scope("candidate_sources") val budget: Duration = target.params(PromotedAccountsFlowParams.FetchCandidateSourceBudget) val candidateSources = Seq( promotedAccountsCandidateSource .mapKeys[PromotedAccountsFlowRequest](r => Seq(r.toAdsRequest(fetchProductionPromotedAccounts))) .mapValue(PromotedAccountsUtil.toCandidateUser) ).map { candidateSource => candidateSource .failOpenWithin(budget, candidateSourceStats).observe(candidateSourceStats) } candidateSources } protected override def preRankerCandidateFilter: Predicate[ (PromotedAccountsFlowRequest, CandidateUser) ] = { val preRankerFilterStats = statsReceiver.scope("pre_ranker") ExcludedUserIdPredicate.observe(preRankerFilterStats.scope("exclude_user_id_predicate")) } /** * rank the candidates */ protected override def selectRanker( target: PromotedAccountsFlowRequest ): Ranker[PromotedAccountsFlowRequest, CandidateUser] = { new IdentityRanker[PromotedAccountsFlowRequest, CandidateUser] } /** * transform the candidates after ranking (e.g. dedupping, grouping and etc) */ protected override def postRankerTransform: Transform[ PromotedAccountsFlowRequest, CandidateUser ] = { new IdentityTransform[PromotedAccountsFlowRequest, CandidateUser] } /** * filter invalid candidates before returning the results. * * Some heavy filters e.g. SGS filter could be applied in this step */ protected override def validateCandidates: Predicate[ (PromotedAccountsFlowRequest, CandidateUser) ] = { new TruePredicate[(PromotedAccountsFlowRequest, CandidateUser)] } /** * transform the candidates into results and return */ protected override def transformResults: Transform[PromotedAccountsFlowRequest, CandidateUser] = { trackingTokenTransform } /** * configuration for recommendation results */ protected override def resultsConfig( target: PromotedAccountsFlowRequest ): RecommendationResultsConfig = { RecommendationResultsConfig( target.params(PromotedAccountsFlowParams.ResultSizeParam), target.params(PromotedAccountsFlowParams.BatchSizeParam) ) } override val statsReceiver: StatsReceiver = baseStatsReceiver.scope("promoted_accounts_flow") } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowParams.scala ================================================ package com.twitter.follow_recommendations.flows.ads import com.twitter.conversions.DurationOps._ import com.twitter.timelines.configapi.Param import com.twitter.util.Duration abstract class PromotedAccountsFlowParams[A](default: A) extends Param[A](default) { override val statName: String = "ads/" + this.getClass.getSimpleName } object PromotedAccountsFlowParams { // number of total slots returned to the end user, available to put ads case object TargetEligibility extends PromotedAccountsFlowParams[Boolean](true) case object ResultSizeParam extends PromotedAccountsFlowParams[Int](Int.MaxValue) case object BatchSizeParam extends PromotedAccountsFlowParams[Int](Int.MaxValue) case object FetchCandidateSourceBudget extends PromotedAccountsFlowParams[Duration](1000.millisecond) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowRequest.scala ================================================ package com.twitter.follow_recommendations.flows.ads import com.twitter.follow_recommendations.common.clients.adserver.AdRequest import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.common.models.HasExcludedUserIds import com.twitter.product_mixer.core.model.marshalling.request.ClientContext import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.timelines.configapi.HasParams import com.twitter.timelines.configapi.Params case class PromotedAccountsFlowRequest( override val clientContext: ClientContext, override val params: Params, displayLocation: DisplayLocation, profileId: Option[Long], // note we also add userId and profileId to excludeUserIds excludeIds: Seq[Long]) extends HasParams with HasClientContext with HasExcludedUserIds with HasDisplayLocation { def toAdsRequest(fetchProductionPromotedAccounts: Boolean): AdRequest = { AdRequest( clientContext = clientContext, displayLocation = displayLocation, isTest = Some(!fetchProductionPromotedAccounts), profileUserId = profileId ) } override val excludedUserIds: Seq[Long] = { excludeIds ++ clientContext.userId.toSeq ++ profileId.toSeq } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsUtil.scala ================================================ package com.twitter.follow_recommendations.flows.ads import com.twitter.follow_recommendations.common.candidate_sources.promoted_accounts.PromotedCandidateUser import com.twitter.follow_recommendations.common.models.AccountProof import com.twitter.follow_recommendations.common.models.AdMetadata import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.Reason import com.twitter.follow_recommendations.common.models.UserCandidateSourceDetails object PromotedAccountsUtil { def toCandidateUser(promotedCandidateUser: PromotedCandidateUser): CandidateUser = { CandidateUser( id = promotedCandidateUser.id, score = None, adMetadata = Some(AdMetadata(promotedCandidateUser.position, promotedCandidateUser.adImpression)), reason = Some( Reason( accountProof = Some(AccountProof(followProof = Some(promotedCandidateUser.followProof)))) ), userCandidateSourceDetails = Some( UserCandidateSourceDetails( promotedCandidateUser.primaryCandidateSource, Map.empty, Map.empty, None)) ) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlow.scala ================================================ package com.twitter.follow_recommendations.flows.content_recommender_flow import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource import com.twitter.follow_recommendations.common.base.GatedPredicateBase import com.twitter.follow_recommendations.common.base.ParamPredicate import com.twitter.follow_recommendations.common.base.Predicate import com.twitter.follow_recommendations.common.base.Ranker import com.twitter.follow_recommendations.common.base.RecommendationFlow import com.twitter.follow_recommendations.common.base.RecommendationResultsConfig import com.twitter.follow_recommendations.common.base.Transform import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.predicates.ExcludedUserIdPredicate import com.twitter.follow_recommendations.common.predicates.InactivePredicate import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicate import com.twitter.follow_recommendations.common.predicates.sgs.InvalidRelationshipPredicate import com.twitter.follow_recommendations.common.predicates.sgs.InvalidTargetCandidateRelationshipTypesPredicate import com.twitter.follow_recommendations.common.predicates.sgs.RecentFollowingPredicate import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.WeightedCandidateSourceRanker import com.twitter.follow_recommendations.common.transforms.dedup.DedupTransform import com.twitter.follow_recommendations.common.transforms.tracking_token.TrackingTokenTransform import com.twitter.follow_recommendations.utils.CandidateSourceHoldbackUtil import com.twitter.follow_recommendations.utils.RecommendationFlowBaseSideEffectsUtil import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.quality_factor.BoundsWithDefault import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactor import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorConfig import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorObserver import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver import javax.inject.Inject import javax.inject.Singleton @Singleton class ContentRecommenderFlow @Inject() ( contentRecommenderFlowCandidateSourceRegistry: ContentRecommenderFlowCandidateSourceRegistry, recentFollowingPredicate: RecentFollowingPredicate, gizmoduckPredicate: GizmoduckPredicate, inactivePredicate: InactivePredicate, sgsPredicate: InvalidTargetCandidateRelationshipTypesPredicate, invalidRelationshipPredicate: InvalidRelationshipPredicate, trackingTokenTransform: TrackingTokenTransform, baseStatsReceiver: StatsReceiver) extends RecommendationFlow[ContentRecommenderRequest, CandidateUser] with RecommendationFlowBaseSideEffectsUtil[ContentRecommenderRequest, CandidateUser] with CandidateSourceHoldbackUtil { override val statsReceiver: StatsReceiver = baseStatsReceiver.scope("content_recommender_flow") override val qualityFactorObserver: Option[QualityFactorObserver] = { val config = LinearLatencyQualityFactorConfig( qualityFactorBounds = BoundsWithDefault(minInclusive = 0.1, maxInclusive = 1.0, default = 1.0), initialDelay = 60.seconds, targetLatency = 100.milliseconds, targetLatencyPercentile = 95.0, delta = 0.001 ) val qualityFactor = LinearLatencyQualityFactor(config) val observer = LinearLatencyQualityFactorObserver(qualityFactor) statsReceiver.provideGauge("quality_factor")(qualityFactor.currentValue.toFloat) Some(observer) } protected override def targetEligibility: Predicate[ContentRecommenderRequest] = new ParamPredicate[ContentRecommenderRequest]( ContentRecommenderParams.TargetEligibility ) protected override def candidateSources( target: ContentRecommenderRequest ): Seq[CandidateSource[ContentRecommenderRequest, CandidateUser]] = { import EnrichedCandidateSource._ val identifiers = ContentRecommenderFlowCandidateSourceWeights.getWeights(target.params).keySet val selected = contentRecommenderFlowCandidateSourceRegistry.select(identifiers) val budget = target.params(ContentRecommenderParams.FetchCandidateSourceBudgetInMillisecond).millisecond filterCandidateSources(target, selected.map(c => c.failOpenWithin(budget, statsReceiver)).toSeq) } protected override val preRankerCandidateFilter: Predicate[ (ContentRecommenderRequest, CandidateUser) ] = { val preRankerFilterStats = statsReceiver.scope("pre_ranker") val recentFollowingPredicateStats = preRankerFilterStats.scope("recent_following_predicate") val invalidRelationshipPredicateStats = preRankerFilterStats.scope("invalid_relationship_predicate") object recentFollowingGatedPredicate extends GatedPredicateBase[(ContentRecommenderRequest, CandidateUser)]( recentFollowingPredicate, recentFollowingPredicateStats ) { override def gate(item: (ContentRecommenderRequest, CandidateUser)): Boolean = item._1.params(ContentRecommenderParams.EnableRecentFollowingPredicate) } object invalidRelationshipGatedPredicate extends GatedPredicateBase[(ContentRecommenderRequest, CandidateUser)]( invalidRelationshipPredicate, invalidRelationshipPredicateStats ) { override def gate(item: (ContentRecommenderRequest, CandidateUser)): Boolean = item._1.params(ContentRecommenderParams.EnableInvalidRelationshipPredicate) } ExcludedUserIdPredicate .observe(preRankerFilterStats.scope("exclude_user_id_predicate")) .andThen(recentFollowingGatedPredicate.observe(recentFollowingPredicateStats)) .andThen(invalidRelationshipGatedPredicate.observe(invalidRelationshipPredicateStats)) } /** * rank the candidates */ protected override def selectRanker( target: ContentRecommenderRequest ): Ranker[ContentRecommenderRequest, CandidateUser] = { val rankersStatsReceiver = statsReceiver.scope("rankers") WeightedCandidateSourceRanker .build[ContentRecommenderRequest]( ContentRecommenderFlowCandidateSourceWeights.getWeights(target.params), randomSeed = target.getRandomizationSeed ).observe(rankersStatsReceiver.scope("weighted_candidate_source_ranker")) } /** * transform the candidates after ranking */ protected override def postRankerTransform: Transform[ ContentRecommenderRequest, CandidateUser ] = { new DedupTransform[ContentRecommenderRequest, CandidateUser] .observe(statsReceiver.scope("dedupping")) } protected override def validateCandidates: Predicate[ (ContentRecommenderRequest, CandidateUser) ] = { val stats = statsReceiver.scope("validate_candidates") val gizmoduckPredicateStats = stats.scope("gizmoduck_predicate") val inactivePredicateStats = stats.scope("inactive_predicate") val sgsPredicateStats = stats.scope("sgs_predicate") val includeGizmoduckPredicate = new ParamPredicate[ContentRecommenderRequest]( ContentRecommenderParams.EnableGizmoduckPredicate) .map[(ContentRecommenderRequest, CandidateUser)] { case (request: ContentRecommenderRequest, _) => request } val includeInactivePredicate = new ParamPredicate[ContentRecommenderRequest]( ContentRecommenderParams.EnableInactivePredicate) .map[(ContentRecommenderRequest, CandidateUser)] { case (request: ContentRecommenderRequest, _) => request } val includeInvalidTargetCandidateRelationshipTypesPredicate = new ParamPredicate[ContentRecommenderRequest]( ContentRecommenderParams.EnableInvalidTargetCandidateRelationshipPredicate) .map[(ContentRecommenderRequest, CandidateUser)] { case (request: ContentRecommenderRequest, _) => request } Predicate .andConcurrently[(ContentRecommenderRequest, CandidateUser)]( Seq( gizmoduckPredicate.observe(gizmoduckPredicateStats).gate(includeGizmoduckPredicate), inactivePredicate.observe(inactivePredicateStats).gate(includeInactivePredicate), sgsPredicate .observe(sgsPredicateStats).gate( includeInvalidTargetCandidateRelationshipTypesPredicate), ) ) } /** * transform the candidates into results and return */ protected override def transformResults: Transform[ContentRecommenderRequest, CandidateUser] = { trackingTokenTransform } /** * configuration for recommendation results */ protected override def resultsConfig( target: ContentRecommenderRequest ): RecommendationResultsConfig = { RecommendationResultsConfig( target.maxResults.getOrElse(target.params(ContentRecommenderParams.ResultSizeParam)), target.params(ContentRecommenderParams.BatchSizeParam) ) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceRegistry.scala ================================================ package com.twitter.follow_recommendations.flows.content_recommender_flow import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.CandidateSourceRegistry import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsSource import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsSource import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonV2Source import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsSource import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource import com.twitter.follow_recommendations.common.candidate_sources.socialgraph.RecentFollowingRecentFollowingExpansionSource import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphCandidateSource import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import javax.inject.Inject import javax.inject.Singleton @Singleton class ContentRecommenderFlowCandidateSourceRegistry @Inject() ( // social based forwardPhoneBookSource: ForwardPhoneBookSource, forwardEmailBookSource: ForwardEmailBookSource, reversePhoneBookSource: ReversePhoneBookSource, reverseEmailBookSource: ReverseEmailBookSource, offlineStrongTiePredictionSource: OfflineStrongTiePredictionSource, triangularLoopsSource: TriangularLoopsSource, userUserGraphCandidateSource: UserUserGraphCandidateSource, realGraphOonSource: RealGraphOonV2Source, recentFollowingRecentFollowingExpansionSource: RecentFollowingRecentFollowingExpansionSource, // activity based recentFollowingSimilarUsersSource: RecentFollowingSimilarUsersSource, recentEngagementSimilarUsersSource: RecentEngagementSimilarUsersSource, repeatedProfileVisitsSource: RepeatedProfileVisitsSource, // geo based popCountrySource: PopCountrySource, popGeohashSource: PopGeohashSource, popCountryBackFillSource: PopCountryBackFillSource, crowdSearchAccountsSource: CrowdSearchAccountsSource, topOrganicFollowsAccountsSource: TopOrganicFollowsAccountsSource, ppmiLocaleFollowSource: PPMILocaleFollowSource, baseStatsReceiver: StatsReceiver) extends CandidateSourceRegistry[ContentRecommenderRequest, CandidateUser] { override val statsReceiver = baseStatsReceiver .scope("content_recommender_flow", "candidate_sources") override val sources: Set[CandidateSource[ContentRecommenderRequest, CandidateUser]] = Seq( forwardPhoneBookSource, forwardEmailBookSource, reversePhoneBookSource, reverseEmailBookSource, offlineStrongTiePredictionSource, triangularLoopsSource, userUserGraphCandidateSource, realGraphOonSource, recentFollowingRecentFollowingExpansionSource, recentFollowingSimilarUsersSource, recentEngagementSimilarUsersSource, repeatedProfileVisitsSource, popCountrySource, popGeohashSource, popCountryBackFillSource, crowdSearchAccountsSource, topOrganicFollowsAccountsSource, ppmiLocaleFollowSource, ).toSet } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeights.scala ================================================ package com.twitter.follow_recommendations.flows.content_recommender_flow import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonV2Source import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsSource import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphCandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsSource import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource import com.twitter.follow_recommendations.common.candidate_sources.socialgraph.RecentFollowingRecentFollowingExpansionSource import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsSource import com.twitter.timelines.configapi.Params object ContentRecommenderFlowCandidateSourceWeights { def getWeights( params: Params ): Map[CandidateSourceIdentifier, Double] = { Map[CandidateSourceIdentifier, Double]( // Social based UserUserGraphCandidateSource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.UserUserGraphSourceWeight), ForwardPhoneBookSource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.ForwardPhoneBookSourceWeight), ReversePhoneBookSource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.ReversePhoneBookSourceWeight), ForwardEmailBookSource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.ForwardEmailBookSourceWeight), ReverseEmailBookSource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.ReverseEmailBookSourceWeight), TriangularLoopsSource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.TriangularLoopsSourceWeight), OfflineStrongTiePredictionSource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.OfflineStrongTiePredictionSourceWeight), RecentFollowingRecentFollowingExpansionSource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.NewFollowingNewFollowingExpansionSourceWeight), RecentFollowingSimilarUsersSource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.NewFollowingSimilarUserSourceWeight), // Activity based RealGraphOonV2Source.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.RealGraphOonSourceWeight), RecentEngagementSimilarUsersSource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.RecentEngagementSimilarUserSourceWeight), RepeatedProfileVisitsSource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.RepeatedProfileVisitsSourceWeight), // Geo based PopCountrySource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.PopCountrySourceWeight), PopGeohashSource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.PopGeohashSourceWeight), PopCountryBackFillSource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.PopCountryBackfillSourceWeight), PPMILocaleFollowSource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.PPMILocaleFollowSourceWeight), CrowdSearchAccountsSource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.CrowdSearchAccountSourceWeight), TopOrganicFollowsAccountsSource.Identifier -> params( ContentRecommenderFlowCandidateSourceWeightsParams.TopOrganicFollowsAccountsSourceWeight), ) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeightsParams.scala ================================================ package com.twitter.follow_recommendations.flows.content_recommender_flow import com.twitter.timelines.configapi.FSBoundedParam object ContentRecommenderFlowCandidateSourceWeightsParams { // Social based case object ForwardPhoneBookSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.ForwardPhoneBookSourceWeight, 1d, 0d, 1000d) case object ForwardEmailBookSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.ForwardEmailBookSourceWeight, 1d, 0d, 1000d) case object ReversePhoneBookSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.ReversePhoneBookSourceWeight, 1d, 0d, 1000d) case object ReverseEmailBookSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.ReverseEmailBookSourceWeight, 1d, 0d, 1000d) case object OfflineStrongTiePredictionSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.OfflineStrongTiePredictionSourceWeight, 1d, 0d, 1000d) case object TriangularLoopsSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.TriangularLoopsSourceWeight, 1d, 0d, 1000d) case object UserUserGraphSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.UserUserGraphSourceWeight, 1d, 0d, 1000d) case object NewFollowingNewFollowingExpansionSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.NewFollowingNewFollowingExpansionSourceWeight, 1d, 0d, 1000d) // Activity based case object NewFollowingSimilarUserSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.NewFollowingSimilarUserSourceWeight, 1d, 0d, 1000d) case object RecentEngagementSimilarUserSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.RecentEngagementSimilarUserSourceWeight, 1d, 0d, 1000d) case object RepeatedProfileVisitsSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.RepeatedProfileVisitsSourceWeight, 1d, 0d, 1000d) case object RealGraphOonSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.RealGraphOonSourceWeight, 1d, 0d, 1000d) // Geo based case object PopCountrySourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.PopCountrySourceWeight, 1d, 0d, 1000d) case object PopGeohashSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.PopGeohashSourceWeight, 1d, 0d, 1000d) case object PopCountryBackfillSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.PopCountryBackfillSourceWeight, 1d, 0d, 1000d) case object PPMILocaleFollowSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.PPMILocaleFollowSourceWeight, 1d, 0d, 1000d) case object TopOrganicFollowsAccountsSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.TopOrganicFollowsAccountsSourceWeight, 1d, 0d, 1000d) case object CrowdSearchAccountSourceWeight extends FSBoundedParam[Double]( ContentRecommenderFlowFeatureSwitchKeys.CrowdSearchAccountSourceWeight, 1d, 0d, 1000d) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFSConfig.scala ================================================ package com.twitter.follow_recommendations.flows.content_recommender_flow import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.Param import javax.inject.Inject import javax.inject.Singleton @Singleton class ContentRecommenderFlowFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( ContentRecommenderParams.IncludeActivityBasedCandidateSource, ContentRecommenderParams.IncludeSocialBasedCandidateSource, ContentRecommenderParams.IncludeGeoBasedCandidateSource, ContentRecommenderParams.IncludeHomeTimelineTweetRecsCandidateSource, ContentRecommenderParams.IncludeSocialProofEnforcedCandidateSource, ContentRecommenderParams.EnableRecentFollowingPredicate, ContentRecommenderParams.EnableGizmoduckPredicate, ContentRecommenderParams.EnableInactivePredicate, ContentRecommenderParams.EnableInvalidTargetCandidateRelationshipPredicate, ContentRecommenderParams.IncludeNewFollowingNewFollowingExpansionCandidateSource, ContentRecommenderParams.IncludeMoreGeoBasedCandidateSource, ContentRecommenderParams.TargetEligibility, ContentRecommenderParams.GetFollowersFromSgs, ContentRecommenderParams.EnableInvalidRelationshipPredicate, ) override val intFSParams: Seq[FSBoundedParam[Int]] = Seq( ContentRecommenderParams.ResultSizeParam, ContentRecommenderParams.BatchSizeParam, ContentRecommenderParams.FetchCandidateSourceBudgetInMillisecond, ContentRecommenderParams.RecentFollowingPredicateBudgetInMillisecond, ) override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( ContentRecommenderFlowCandidateSourceWeightsParams.ForwardPhoneBookSourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.ForwardEmailBookSourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.ReversePhoneBookSourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.ReverseEmailBookSourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.OfflineStrongTiePredictionSourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.TriangularLoopsSourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.UserUserGraphSourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.NewFollowingNewFollowingExpansionSourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.NewFollowingSimilarUserSourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.RecentEngagementSimilarUserSourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.RepeatedProfileVisitsSourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.RealGraphOonSourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.PopCountrySourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.PopGeohashSourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.PopCountryBackfillSourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.PPMILocaleFollowSourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.TopOrganicFollowsAccountsSourceWeight, ContentRecommenderFlowCandidateSourceWeightsParams.CrowdSearchAccountSourceWeight, ) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFeatureSwitchKeys.scala ================================================ package com.twitter.follow_recommendations.flows.content_recommender_flow object ContentRecommenderFlowFeatureSwitchKeys { val TargetUserEligible = "content_recommender_flow_target_eligible" val ResultSize = "content_recommender_flow_result_size" val BatchSize = "content_recommender_flow_batch_size" val RecentFollowingPredicateBudgetInMillisecond = "content_recommender_flow_recent_following_predicate_budget_in_ms" val CandidateGenerationBudgetInMillisecond = "content_recommender_flow_candidate_generation_budget_in_ms" val EnableRecentFollowingPredicate = "content_recommender_flow_enable_recent_following_predicate" val EnableGizmoduckPredicate = "content_recommender_flow_enable_gizmoduck_predicate" val EnableInactivePredicate = "content_recommender_flow_enable_inactive_predicate" val EnableInvalidTargetCandidateRelationshipPredicate = "content_recommender_flow_enable_invalid_target_candidate_relationship_predicate" val IncludeActivityBasedCandidateSource = "content_recommender_flow_include_activity_based_candidate_source" val IncludeSocialBasedCandidateSource = "content_recommender_flow_include_social_based_candidate_source" val IncludeGeoBasedCandidateSource = "content_recommender_flow_include_geo_based_candidate_source" val IncludeHomeTimelineTweetRecsCandidateSource = "content_recommender_flow_include_home_timeline_tweet_recs_candidate_source" val IncludeSocialProofEnforcedCandidateSource = "content_recommender_flow_include_social_proof_enforced_candidate_source" val IncludeNewFollowingNewFollowingExpansionCandidateSource = "content_recommender_flow_include_new_following_new_following_expansion_candidate_source" val IncludeMoreGeoBasedCandidateSource = "content_recommender_flow_include_more_geo_based_candidate_source" val GetFollowersFromSgs = "content_recommender_flow_get_followers_from_sgs" val EnableInvalidRelationshipPredicate = "content_recommender_flow_enable_invalid_relationship_predicate" // Candidate source weight param keys // Social based val ForwardPhoneBookSourceWeight = "content_recommender_flow_candidate_source_weight_forward_phone_book" val ForwardEmailBookSourceWeight = "content_recommender_flow_candidate_source_weight_forward_email_book" val ReversePhoneBookSourceWeight = "content_recommender_flow_candidate_source_weight_reverse_phone_book" val ReverseEmailBookSourceWeight = "content_recommender_flow_candidate_source_weight_reverse_email_book" val OfflineStrongTiePredictionSourceWeight = "content_recommender_flow_candidate_source_weight_offline_stp" val TriangularLoopsSourceWeight = "content_recommender_flow_candidate_source_weight_triangular_loops" val UserUserGraphSourceWeight = "content_recommender_flow_candidate_source_weight_user_user_graph" val NewFollowingNewFollowingExpansionSourceWeight = "content_recommender_flow_candidate_source_weight_new_following_new_following_expansion" // Activity based val NewFollowingSimilarUserSourceWeight = "content_recommender_flow_candidate_source_weight_new_following_similar_user" val RecentEngagementSimilarUserSourceWeight = "content_recommender_flow_candidate_source_weight_recent_engagement_similar_user" val RepeatedProfileVisitsSourceWeight = "content_recommender_flow_candidate_source_weight_repeated_profile_visits" val RealGraphOonSourceWeight = "content_recommender_flow_candidate_source_weight_real_graph_oon" // Geo based val PopCountrySourceWeight = "content_recommender_flow_candidate_source_weight_pop_country" val PopGeohashSourceWeight = "content_recommender_flow_candidate_source_weight_pop_geohash" val PopCountryBackfillSourceWeight = "content_recommender_flow_candidate_source_weight_pop_country_backfill" val PPMILocaleFollowSourceWeight = "content_recommender_flow_candidate_source_weight_ppmi_locale_follow" val TopOrganicFollowsAccountsSourceWeight = "content_recommender_flow_candidate_source_weight_top_organic_follow_account" val CrowdSearchAccountSourceWeight = "content_recommender_flow_candidate_source_weight_crowd_search_account" } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderParams.scala ================================================ package com.twitter.follow_recommendations.flows.content_recommender_flow import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.Param abstract class ContentRecommenderParams[A](default: A) extends Param[A](default) { override val statName: String = "content_recommender/" + this.getClass.getSimpleName } object ContentRecommenderParams { case object TargetEligibility extends FSParam[Boolean](ContentRecommenderFlowFeatureSwitchKeys.TargetUserEligible, true) case object ResultSizeParam extends FSBoundedParam[Int](ContentRecommenderFlowFeatureSwitchKeys.ResultSize, 15, 1, 500) case object BatchSizeParam extends FSBoundedParam[Int](ContentRecommenderFlowFeatureSwitchKeys.BatchSize, 15, 1, 500) case object RecentFollowingPredicateBudgetInMillisecond extends FSBoundedParam[Int]( ContentRecommenderFlowFeatureSwitchKeys.RecentFollowingPredicateBudgetInMillisecond, 8, 1, 50) case object FetchCandidateSourceBudgetInMillisecond extends FSBoundedParam[Int]( ContentRecommenderFlowFeatureSwitchKeys.CandidateGenerationBudgetInMillisecond, 60, 1, 80) case object EnableRecentFollowingPredicate extends FSParam[Boolean]( ContentRecommenderFlowFeatureSwitchKeys.EnableRecentFollowingPredicate, true) case object EnableGizmoduckPredicate extends FSParam[Boolean]( ContentRecommenderFlowFeatureSwitchKeys.EnableGizmoduckPredicate, false) case object EnableInactivePredicate extends FSParam[Boolean]( ContentRecommenderFlowFeatureSwitchKeys.EnableInactivePredicate, false) case object EnableInvalidTargetCandidateRelationshipPredicate extends FSParam[Boolean]( ContentRecommenderFlowFeatureSwitchKeys.EnableInvalidTargetCandidateRelationshipPredicate, false) case object IncludeActivityBasedCandidateSource extends FSParam[Boolean]( ContentRecommenderFlowFeatureSwitchKeys.IncludeActivityBasedCandidateSource, true) case object IncludeSocialBasedCandidateSource extends FSParam[Boolean]( ContentRecommenderFlowFeatureSwitchKeys.IncludeSocialBasedCandidateSource, true) case object IncludeGeoBasedCandidateSource extends FSParam[Boolean]( ContentRecommenderFlowFeatureSwitchKeys.IncludeGeoBasedCandidateSource, true) case object IncludeHomeTimelineTweetRecsCandidateSource extends FSParam[Boolean]( ContentRecommenderFlowFeatureSwitchKeys.IncludeHomeTimelineTweetRecsCandidateSource, false) case object IncludeSocialProofEnforcedCandidateSource extends FSParam[Boolean]( ContentRecommenderFlowFeatureSwitchKeys.IncludeSocialProofEnforcedCandidateSource, false) case object IncludeNewFollowingNewFollowingExpansionCandidateSource extends FSParam[Boolean]( ContentRecommenderFlowFeatureSwitchKeys.IncludeNewFollowingNewFollowingExpansionCandidateSource, false) case object IncludeMoreGeoBasedCandidateSource extends FSParam[Boolean]( ContentRecommenderFlowFeatureSwitchKeys.IncludeMoreGeoBasedCandidateSource, false) case object GetFollowersFromSgs extends FSParam[Boolean](ContentRecommenderFlowFeatureSwitchKeys.GetFollowersFromSgs, false) case object EnableInvalidRelationshipPredicate extends FSParam[Boolean]( ContentRecommenderFlowFeatureSwitchKeys.EnableInvalidRelationshipPredicate, false) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequest.scala ================================================ package com.twitter.follow_recommendations.flows.content_recommender_flow import com.twitter.core_workflows.user_model.thriftscala.UserState import com.twitter.follow_recommendations.common.models.DebugOptions import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode import com.twitter.follow_recommendations.common.models.HasDebugOptions import com.twitter.follow_recommendations.common.models.HasDisplayLocation import com.twitter.follow_recommendations.common.models.HasExcludedUserIds import com.twitter.follow_recommendations.common.models.HasGeohashAndCountryCode import com.twitter.follow_recommendations.common.models.HasInvalidRelationshipUserIds import com.twitter.follow_recommendations.common.models.HasRecentFollowedByUserIds import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds import com.twitter.follow_recommendations.common.models.HasUserState import com.twitter.product_mixer.core.model.marshalling.request.ClientContext import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.timelines.configapi.HasParams import com.twitter.timelines.configapi.Params case class ContentRecommenderRequest( override val params: Params, override val clientContext: ClientContext, inputExcludeUserIds: Seq[Long], override val recentFollowedUserIds: Option[Seq[Long]], override val recentFollowedByUserIds: Option[Seq[Long]], override val invalidRelationshipUserIds: Option[Set[Long]], override val displayLocation: DisplayLocation, maxResults: Option[Int] = None, override val debugOptions: Option[DebugOptions] = None, override val geohashAndCountryCode: Option[GeohashAndCountryCode] = None, override val userState: Option[UserState] = None) extends HasParams with HasClientContext with HasDisplayLocation with HasDebugOptions with HasRecentFollowedUserIds with HasRecentFollowedByUserIds with HasInvalidRelationshipUserIds with HasExcludedUserIds with HasUserState with HasGeohashAndCountryCode { override val excludedUserIds: Seq[Long] = { inputExcludeUserIds ++ clientContext.userId.toSeq } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequestBuilder.scala ================================================ package com.twitter.follow_recommendations.flows.content_recommender_flow import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.clients.geoduck.UserLocationFetcher import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient import com.twitter.follow_recommendations.common.clients.user_state.UserStateClient import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueOptionalWithStats import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStats import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStatsWithin import com.twitter.follow_recommendations.products.common.ProductRequest import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class ContentRecommenderRequestBuilder @Inject() ( socialGraph: SocialGraphClient, userLocationFetcher: UserLocationFetcher, userStateClient: UserStateClient, statsReceiver: StatsReceiver) { val stats: StatsReceiver = statsReceiver.scope("content_recommender_request_builder") val invalidRelationshipUsersStats: StatsReceiver = stats.scope("invalidRelationshipUserIds") private val invalidRelationshipUsersMaxSizeCounter = invalidRelationshipUsersStats.counter("maxSize") private val invalidRelationshipUsersNotMaxSizeCounter = invalidRelationshipUsersStats.counter("notMaxSize") def build(req: ProductRequest): Stitch[ContentRecommenderRequest] = { val userStateStitch = Stitch .collect(req.recommendationRequest.clientContext.userId.map(userId => userStateClient.getUserState(userId))).map(_.flatten) val recentFollowedUserIdsStitch = Stitch .collect(req.recommendationRequest.clientContext.userId.map { userId => rescueWithStatsWithin( socialGraph.getRecentFollowedUserIds(userId), stats, "recentFollowedUserIds", req .params( ContentRecommenderParams.RecentFollowingPredicateBudgetInMillisecond).millisecond ) }) val recentFollowedByUserIdsStitch = if (req.params(ContentRecommenderParams.GetFollowersFromSgs)) { Stitch .collect( req.recommendationRequest.clientContext.userId.map(userId => rescueWithStatsWithin( socialGraph.getRecentFollowedByUserIdsFromCachedColumn(userId), stats, "recentFollowedByUserIds", req .params(ContentRecommenderParams.RecentFollowingPredicateBudgetInMillisecond) .millisecond ))) } else Stitch.None val invalidRelationshipUserIdsStitch: Stitch[Option[Seq[Long]]] = if (req.params(ContentRecommenderParams.EnableInvalidRelationshipPredicate)) { Stitch .collect( req.recommendationRequest.clientContext.userId.map { userId => rescueWithStats( socialGraph .getInvalidRelationshipUserIdsFromCachedColumn(userId) .onSuccess(ids => if (ids.size >= SocialGraphClient.MaxNumInvalidRelationship) { invalidRelationshipUsersMaxSizeCounter.incr() } else { invalidRelationshipUsersNotMaxSizeCounter.incr() }), stats, "invalidRelationshipUserIds" ) } ) } else { Stitch.None } val locationStitch = rescueOptionalWithStats( userLocationFetcher.getGeohashAndCountryCode( req.recommendationRequest.clientContext.userId, req.recommendationRequest.clientContext.ipAddress ), stats, "userLocation" ) Stitch .join( recentFollowedUserIdsStitch, recentFollowedByUserIdsStitch, invalidRelationshipUserIdsStitch, locationStitch, userStateStitch) .map { case ( recentFollowedUserIds, recentFollowedByUserIds, invalidRelationshipUserIds, location, userState) => ContentRecommenderRequest( req.params, req.recommendationRequest.clientContext, req.recommendationRequest.excludedIds.getOrElse(Nil), recentFollowedUserIds, recentFollowedByUserIds, invalidRelationshipUserIds.map(_.toSet), req.recommendationRequest.displayLocation, req.recommendationRequest.maxResults, req.recommendationRequest.debugParams.flatMap(_.debugOptions), location, userState ) } } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceRegistry.scala ================================================ package com.twitter.follow_recommendations.flows.post_nux_ml import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.CandidateSourceRegistry import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsSource import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsSource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashQualityFollowSource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonV2Source import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RecentEngagementNonDirectFollowSource import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsSource import com.twitter.follow_recommendations.common.candidate_sources.salsa.RecentEngagementDirectFollowSalsaExpansionSource import com.twitter.follow_recommendations.common.candidate_sources.sims.LinearRegressionFollow2vecNearestNeighborsStore import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource import com.twitter.follow_recommendations.common.candidate_sources.stp.OnlineSTPSourceScorer import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphCandidateSource import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import javax.inject.Inject import javax.inject.Singleton @Singleton class PostNuxMlCandidateSourceRegistry @Inject() ( crowdSearchAccountsCandidateSource: CrowdSearchAccountsSource, topOrganicFollowsAccountsSource: TopOrganicFollowsAccountsSource, linearRegressionfollow2vecNearestNeighborsStore: LinearRegressionFollow2vecNearestNeighborsStore, forwardEmailBookSource: ForwardEmailBookSource, forwardPhoneBookSource: ForwardPhoneBookSource, offlineStrongTiePredictionSource: OfflineStrongTiePredictionSource, onlineSTPSource: OnlineSTPSourceScorer, popCountrySource: PopCountrySource, popCountryBackFillSource: PopCountryBackFillSource, popGeohashSource: PopGeohashSource, recentEngagementDirectFollowSimilarUsersSource: RecentEngagementSimilarUsersSource, recentEngagementNonDirectFollowSource: RecentEngagementNonDirectFollowSource, recentEngagementDirectFollowSalsaExpansionSource: RecentEngagementDirectFollowSalsaExpansionSource, recentFollowingSimilarUsersSource: RecentFollowingSimilarUsersSource, realGraphOonV2Source: RealGraphOonV2Source, repeatedProfileVisitSource: RepeatedProfileVisitsSource, reverseEmailBookSource: ReverseEmailBookSource, reversePhoneBookSource: ReversePhoneBookSource, triangularLoopsSource: TriangularLoopsSource, userUserGraphCandidateSource: UserUserGraphCandidateSource, ppmiLocaleFollowSource: PPMILocaleFollowSource, popGeohashQualityFollowSource: PopGeohashQualityFollowSource, baseStatsReceiver: StatsReceiver, ) extends CandidateSourceRegistry[PostNuxMlRequest, CandidateUser] { import EnrichedCandidateSource._ override val statsReceiver = baseStatsReceiver .scope("post_nux_ml_flow", "candidate_sources") // sources primarily based on social graph signals private[this] val socialSources = Seq( linearRegressionfollow2vecNearestNeighborsStore.mapKeys[PostNuxMlRequest]( _.getOptionalUserId.toSeq), forwardEmailBookSource, forwardPhoneBookSource, offlineStrongTiePredictionSource, onlineSTPSource, reverseEmailBookSource, reversePhoneBookSource, triangularLoopsSource, ) // sources primarily based on geo signals private[this] val geoSources = Seq( popCountrySource, popCountryBackFillSource, popGeohashSource, popGeohashQualityFollowSource, topOrganicFollowsAccountsSource, crowdSearchAccountsCandidateSource, ppmiLocaleFollowSource, ) // sources primarily based on recent activity signals private[this] val activitySources = Seq( repeatedProfileVisitSource, recentEngagementDirectFollowSalsaExpansionSource.mapKeys[PostNuxMlRequest]( _.getOptionalUserId.toSeq), recentEngagementDirectFollowSimilarUsersSource, recentEngagementNonDirectFollowSource.mapKeys[PostNuxMlRequest](_.getOptionalUserId.toSeq), recentFollowingSimilarUsersSource, realGraphOonV2Source, userUserGraphCandidateSource, ) override val sources: Set[CandidateSource[PostNuxMlRequest, CandidateUser]] = ( geoSources ++ socialSources ++ activitySources ).toSet } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceWeightParams.scala ================================================ package com.twitter.follow_recommendations.flows.post_nux_ml import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.Param abstract class PostNuxMlCandidateSourceWeightParams[A](default: A) extends Param[A](default) { override val statName: String = "post_nux_ml/" + this.getClass.getSimpleName } object PostNuxMlCandidateSourceWeightParams { case object CandidateWeightCrowdSearch extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightCrowdSearch, 1.0, 0.0, 1000.0 ) case object CandidateWeightTopOrganicFollow extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightTopOrganicFollow, 1.0, 0.0, 1000.0 ) case object CandidateWeightPPMILocaleFollow extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPPMILocaleFollow, 1.0, 0.0, 1000.0 ) case object CandidateWeightForwardEmailBook extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightForwardEmailBook, 1.0, 0.0, 1000.0 ) case object CandidateWeightForwardPhoneBook extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightForwardPhoneBook, 1.0, 0.0, 1000.0 ) case object CandidateWeightOfflineStrongTiePrediction extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightOfflineStrongTiePrediction, 1.0, 0.0, 1000.0 ) case object CandidateWeightOnlineStp extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightOnlineStp, 1.0, 0.0, 1000.0 ) case object CandidateWeightPopCountry extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPopCountry, 1.0, 0.0, 1000.0 ) case object CandidateWeightPopGeohash extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPopGeohash, 1.0, 0.0, 1000.0 ) case object CandidateWeightPopGeohashQualityFollow extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPopGeohashQualityFollow, 1.0, 0.0, 1000.0 ) case object CandidateWeightPopGeoBackfill extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPopGeoBackfill, 1, 0.0, 1000.0 ) case object CandidateWeightRecentFollowingSimilarUsers extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRecentFollowingSimilarUsers, 1.0, 0.0, 1000.0 ) case object CandidateWeightRecentEngagementDirectFollowSalsaExpansion extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRecentEngagementDirectFollowSalsaExpansion, 1.0, 0.0, 1000.0 ) case object CandidateWeightRecentEngagementNonDirectFollow extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRecentEngagementNonDirectFollow, 1.0, 0.0, 1000.0 ) case object CandidateWeightRecentEngagementSimilarUsers extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRecentEngagementSimilarUsers, 1.0, 0.0, 1000.0 ) case object CandidateWeightRepeatedProfileVisits extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRepeatedProfileVisits, 1.0, 0.0, 1000.0 ) case object CandidateWeightFollow2vecNearestNeighbors extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightFollow2vecNearestNeighbors, 1.0, 0.0, 1000.0 ) case object CandidateWeightReverseEmailBook extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightReverseEmailBook, 1.0, 0.0, 1000.0 ) case object CandidateWeightReversePhoneBook extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightReversePhoneBook, 1.0, 0.0, 1000.0 ) case object CandidateWeightTriangularLoops extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightTriangularLoops, 1.0, 0.0, 1000.0 ) case object CandidateWeightTwoHopRandomWalk extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightTwoHopRandomWalk, 1.0, 0.0, 1000.0 ) case object CandidateWeightUserUserGraph extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightUserUserGraph, 1.0, 0.0, 1000.0 ) case object CandidateWeightRealGraphOonV2 extends FSBoundedParam[Double]( PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRealGraphOonV2, 1.0, 0.0, 2000.0 ) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCombinedRankerBuilder.scala ================================================ package com.twitter.follow_recommendations.flows.post_nux_ml import com.google.inject.Inject import com.google.inject.Singleton import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.IdentityRanker import com.twitter.follow_recommendations.common.base.IdentityTransform import com.twitter.follow_recommendations.common.base.Ranker import com.twitter.follow_recommendations.common.base.Transform import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature import com.twitter.follow_recommendations.common.models._ import com.twitter.follow_recommendations.common.rankers.common.RankerId import com.twitter.follow_recommendations.common.rankers.fatigue_ranker.ImpressionBasedFatigueRanker import com.twitter.follow_recommendations.common.rankers.first_n_ranker.FirstNRanker import com.twitter.follow_recommendations.common.rankers.first_n_ranker.FirstNRankerParams import com.twitter.follow_recommendations.common.rankers.interleave_ranker.InterleaveRanker import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.HydrateFeaturesTransform import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.MlRanker import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.MlRankerParams import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.WeightedCandidateSourceRanker import com.twitter.follow_recommendations.configapi.candidates.HydrateCandidateParamsTransform import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.timelines.configapi.HasParams /** * Used to build the combined ranker comprising 4 stages of ranking: * - weighted sampler * - truncating to the top N merged results for ranking * - ML ranker * - Interleaving ranker for producer-side experiments * - impression-based fatigueing */ @Singleton class PostNuxMlCombinedRankerBuilder[ T <: HasParams with HasSimilarToContext with HasClientContext with HasExcludedUserIds with HasDisplayLocation with HasDebugOptions with HasPreFetchedFeature with HasDismissedUserIds with HasQualityFactor] @Inject() ( firstNRanker: FirstNRanker[T], hydrateFeaturesTransform: HydrateFeaturesTransform[T], hydrateCandidateParamsTransform: HydrateCandidateParamsTransform[T], mlRanker: MlRanker[T], statsReceiver: StatsReceiver) { private[this] val stats: StatsReceiver = statsReceiver.scope("post_nux_ml_ranker") // we construct each ranker independently and chain them together def build( request: T, candidateSourceWeights: Map[CandidateSourceIdentifier, Double] ): Ranker[T, CandidateUser] = { val displayLocationStats = stats.scope(request.displayLocation.toString) val weightedRankerStats: StatsReceiver = displayLocationStats.scope("weighted_candidate_source_ranker") val firstNRankerStats: StatsReceiver = displayLocationStats.scope("first_n_ranker") val hydrateCandidateParamsStats = displayLocationStats.scope("hydrate_candidate_params") val fatigueRankerStats = displayLocationStats.scope("fatigue_ranker") val interleaveRankerStats = displayLocationStats.scope("interleave_ranker") val allRankersStats = displayLocationStats.scope("all_rankers") // Checking if the heavy-ranker is an experimental model. // If it is, InterleaveRanker and candidate parameter hydration are disabled. // *NOTE* that consumer-side experiments should at any time take a small % of traffic, less // than 20% for instance, to leave enough room for producer experiments. Increasing bucket // size for producer experiments lead to other issues and is not a viable option for faster // experiments. val requestRankerId = request.params(MlRankerParams.RequestScorerIdParam) if (requestRankerId != RankerId.PostNuxProdRanker) { hydrateCandidateParamsStats.counter(s"disabled_by_${requestRankerId.toString}").incr() interleaveRankerStats.counter(s"disabled_by_${requestRankerId.toString}").incr() } // weighted ranker that samples from the candidate sources val weightedRanker = WeightedCandidateSourceRanker .build[T]( candidateSourceWeights, request.params(PostNuxMlParams.CandidateShuffler).shuffle(request.getRandomizationSeed), randomSeed = request.getRandomizationSeed ).observe(weightedRankerStats) // ranker that takes the first n results (ie truncates output) while merging duplicates val firstNRankerObs = firstNRanker.observe(firstNRankerStats) // either ML ranker that uses deepbirdv2 to score or no ranking val mainRanker: Ranker[T, CandidateUser] = buildMainRanker(request, requestRankerId == RankerId.PostNuxProdRanker, displayLocationStats) // fatigue ranker that uses wtf impressions to fatigue val fatigueRanker = buildFatigueRanker(request, fatigueRankerStats).observe(fatigueRankerStats) // interleaveRanker combines rankings from several rankers and enforces candidates' ranks in // experiment buckets according to their assigned ranker model. val interleaveRanker = buildInterleaveRanker( request, requestRankerId == RankerId.PostNuxProdRanker, interleaveRankerStats) .observe(interleaveRankerStats) weightedRanker .andThen(firstNRankerObs) .andThen(mainRanker) .andThen(fatigueRanker) .andThen(interleaveRanker) .observe(allRankersStats) } def buildMainRanker( request: T, isMainRankerPostNuxProd: Boolean, displayLocationStats: StatsReceiver ): Ranker[T, CandidateUser] = { // note that we may be disabling heavy ranker for users not bucketed // (due to empty results from the new candidate source) // need a better solution in the future val mlRankerStats = displayLocationStats.scope("ml_ranker") val noMlRankerStats = displayLocationStats.scope("no_ml_ranker") val hydrateFeaturesStats = displayLocationStats.scope("hydrate_features") val hydrateCandidateParamsStats = displayLocationStats.scope("hydrate_candidate_params") val notHydrateCandidateParamsStats = displayLocationStats.scope("not_hydrate_candidate_params") val rankerStats = displayLocationStats.scope("ranker") val mlRankerDisabledByExperimentsCounter = mlRankerStats.counter("disabled_by_experiments") val mlRankerDisabledByQualityFactorCounter = mlRankerStats.counter("disabled_by_quality_factor") val disabledByQualityFactor = request.qualityFactor .exists(_ <= request.params(PostNuxMlParams.TurnoffMLScorerQFThreshold)) if (disabledByQualityFactor) mlRankerDisabledByQualityFactorCounter.incr() if (request.params(PostNuxMlParams.UseMlRanker) && !disabledByQualityFactor) { val hydrateFeatures = hydrateFeaturesTransform .observe(hydrateFeaturesStats) val optionalHydratedParamsTransform: Transform[T, CandidateUser] = { // We disable candidate parameter hydration for experimental heavy-ranker models. if (isMainRankerPostNuxProd && request.params(PostNuxMlParams.EnableCandidateParamHydration)) { hydrateCandidateParamsTransform .observe(hydrateCandidateParamsStats) } else { new IdentityTransform[T, CandidateUser]() .observe(notHydrateCandidateParamsStats) } } val candidateSize = request.params(FirstNRankerParams.CandidatesToRank) Ranker .chain( hydrateFeatures.andThen(optionalHydratedParamsTransform), mlRanker.observe(mlRankerStats), ) .within( request.params(PostNuxMlParams.MlRankerBudget), rankerStats.scope(s"n$candidateSize")) } else { new IdentityRanker[T, CandidateUser].observe(noMlRankerStats) } } def buildInterleaveRanker( request: T, isMainRankerPostNuxProd: Boolean, interleaveRankerStats: StatsReceiver ): Ranker[T, CandidateUser] = { // InterleaveRanker is enabled only for display locations powered by the PostNux heavy-ranker. if (request.params(PostNuxMlParams.EnableInterleaveRanker) && // InterleaveRanker is disabled for requests with experimental heavy-rankers. isMainRankerPostNuxProd) { new InterleaveRanker[T](interleaveRankerStats) } else { new IdentityRanker[T, CandidateUser]() } } def buildFatigueRanker( request: T, fatigueRankerStats: StatsReceiver ): Ranker[T, CandidateUser] = { if (request.params(PostNuxMlParams.EnableFatigueRanker)) { ImpressionBasedFatigueRanker .build[T]( fatigueRankerStats ).within(request.params(PostNuxMlParams.FatigueRankerBudget), fatigueRankerStats) } else { new IdentityRanker[T, CandidateUser]() } } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlow.scala ================================================ package com.twitter.follow_recommendations.flows.post_nux_ml import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource._ import com.twitter.follow_recommendations.common.base._ import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.FilterReason import com.twitter.follow_recommendations.common.predicates.dismiss.DismissedCandidatePredicate import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicate import com.twitter.follow_recommendations.common.transforms.ranker_id.RandomRankerIdTransform import com.twitter.follow_recommendations.common.predicates.sgs.InvalidTargetCandidateRelationshipTypesPredicate import com.twitter.follow_recommendations.common.predicates.sgs.RecentFollowingPredicate import com.twitter.follow_recommendations.common.predicates.CandidateParamPredicate import com.twitter.follow_recommendations.common.predicates.CandidateSourceParamPredicate import com.twitter.follow_recommendations.common.predicates.CuratedCompetitorListPredicate import com.twitter.follow_recommendations.common.predicates.ExcludedUserIdPredicate import com.twitter.follow_recommendations.common.predicates.InactivePredicate import com.twitter.follow_recommendations.common.predicates.PreviouslyRecommendedUserIdsPredicate import com.twitter.follow_recommendations.common.predicates.user_activity.NonNearZeroUserActivityPredicate import com.twitter.follow_recommendations.common.transforms.dedup.DedupTransform import com.twitter.follow_recommendations.common.transforms.modify_social_proof.ModifySocialProofTransform import com.twitter.follow_recommendations.common.transforms.tracking_token.TrackingTokenTransform import com.twitter.follow_recommendations.common.transforms.weighted_sampling.SamplingTransform import com.twitter.follow_recommendations.configapi.candidates.CandidateUserParamsFactory import com.twitter.follow_recommendations.configapi.params.GlobalParams import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableGFSSocialProofTransform import com.twitter.follow_recommendations.utils.CandidateSourceHoldbackUtil import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.timelines.configapi.Params import com.twitter.util.Duration import javax.inject.Inject import javax.inject.Singleton import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient import com.twitter.follow_recommendations.common.predicates.hss.HssPredicate import com.twitter.follow_recommendations.common.predicates.sgs.InvalidRelationshipPredicate import com.twitter.follow_recommendations.common.transforms.modify_social_proof.RemoveAccountProofTransform import com.twitter.follow_recommendations.logging.FrsLogger import com.twitter.follow_recommendations.models.RecommendationFlowData import com.twitter.follow_recommendations.utils.RecommendationFlowBaseSideEffectsUtil import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier import com.twitter.product_mixer.core.quality_factor.BoundsWithDefault import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactor import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorConfig import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorObserver import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver import com.twitter.stitch.Stitch /** * We use this flow for all post-nux display locations that would use a machine-learning-based-ranker * eg HTL, Sidebar, etc * Note that the RankedPostNuxFlow is used primarily for scribing/data collection, and doesn't * incorporate all of the other components in a flow (candidate source generation, predicates etc) */ @Singleton class PostNuxMlFlow @Inject() ( postNuxMlCandidateSourceRegistry: PostNuxMlCandidateSourceRegistry, postNuxMlCombinedRankerBuilder: PostNuxMlCombinedRankerBuilder[PostNuxMlRequest], curatedCompetitorListPredicate: CuratedCompetitorListPredicate, gizmoduckPredicate: GizmoduckPredicate, sgsPredicate: InvalidTargetCandidateRelationshipTypesPredicate, hssPredicate: HssPredicate, invalidRelationshipPredicate: InvalidRelationshipPredicate, recentFollowingPredicate: RecentFollowingPredicate, nonNearZeroUserActivityPredicate: NonNearZeroUserActivityPredicate, inactivePredicate: InactivePredicate, dismissedCandidatePredicate: DismissedCandidatePredicate, previouslyRecommendedUserIdsPredicate: PreviouslyRecommendedUserIdsPredicate, modifySocialProofTransform: ModifySocialProofTransform, removeAccountProofTransform: RemoveAccountProofTransform, trackingTokenTransform: TrackingTokenTransform, randomRankerIdTransform: RandomRankerIdTransform, candidateParamsFactory: CandidateUserParamsFactory[PostNuxMlRequest], samplingTransform: SamplingTransform, frsLogger: FrsLogger, baseStatsReceiver: StatsReceiver) extends RecommendationFlow[PostNuxMlRequest, CandidateUser] with RecommendationFlowBaseSideEffectsUtil[PostNuxMlRequest, CandidateUser] with CandidateSourceHoldbackUtil { override protected val targetEligibility: Predicate[PostNuxMlRequest] = new ParamPredicate[PostNuxMlRequest](PostNuxMlParams.TargetEligibility) override val statsReceiver: StatsReceiver = baseStatsReceiver.scope("post_nux_ml_flow") override val qualityFactorObserver: Option[QualityFactorObserver] = { val config = LinearLatencyQualityFactorConfig( qualityFactorBounds = BoundsWithDefault(minInclusive = 0.1, maxInclusive = 1.0, default = 1.0), initialDelay = 60.seconds, targetLatency = 700.milliseconds, targetLatencyPercentile = 95.0, delta = 0.001 ) val qualityFactor = LinearLatencyQualityFactor(config) val observer = LinearLatencyQualityFactorObserver(qualityFactor) statsReceiver.provideGauge("quality_factor")(qualityFactor.currentValue.toFloat) Some(observer) } override protected def updateTarget(request: PostNuxMlRequest): Stitch[PostNuxMlRequest] = { Stitch.value( request.copy(qualityFactor = qualityFactorObserver.map(_.qualityFactor.currentValue)) ) } private[post_nux_ml] def getCandidateSourceIdentifiers( params: Params ): Set[CandidateSourceIdentifier] = { PostNuxMlFlowCandidateSourceWeights.getWeights(params).keySet } override protected def candidateSources( request: PostNuxMlRequest ): Seq[CandidateSource[PostNuxMlRequest, CandidateUser]] = { val identifiers = getCandidateSourceIdentifiers(request.params) val selected: Set[CandidateSource[PostNuxMlRequest, CandidateUser]] = postNuxMlCandidateSourceRegistry.select(identifiers) val budget: Duration = request.params(PostNuxMlParams.FetchCandidateSourceBudget) filterCandidateSources( request, selected.map(c => c.failOpenWithin(budget, statsReceiver)).toSeq) } override protected val preRankerCandidateFilter: Predicate[(PostNuxMlRequest, CandidateUser)] = { val stats = statsReceiver.scope("pre_ranker") object excludeNearZeroUserPredicate extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)]( nonNearZeroUserActivityPredicate, stats.scope("exclude_near_zero_predicate") ) { override def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean = item._1.params(PostNuxMlParams.ExcludeNearZeroCandidates) } object invalidRelationshipGatedPredicate extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)]( invalidRelationshipPredicate, stats.scope("invalid_relationship_predicate") ) { override def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean = item._1.params(PostNuxMlParams.EnableInvalidRelationshipPredicate) } ExcludedUserIdPredicate .observe(stats.scope("exclude_user_id_predicate")) .andThen( recentFollowingPredicate.observe(stats.scope("recent_following_predicate")) ) .andThen( dismissedCandidatePredicate.observe(stats.scope("dismissed_candidate_predicate")) ) .andThen( previouslyRecommendedUserIdsPredicate.observe( stats.scope("previously_recommended_user_ids_predicate")) ) .andThen( invalidRelationshipGatedPredicate.observe(stats.scope("invalid_relationship_predicate")) ) .andThen( excludeNearZeroUserPredicate.observe(stats.scope("exclude_near_zero_user_state")) ) .observe(stats.scope("overall_pre_ranker_candidate_filter")) } override protected def selectRanker( request: PostNuxMlRequest ): Ranker[PostNuxMlRequest, CandidateUser] = { postNuxMlCombinedRankerBuilder.build( request, PostNuxMlFlowCandidateSourceWeights.getWeights(request.params)) } override protected val postRankerTransform: Transform[PostNuxMlRequest, CandidateUser] = { new DedupTransform[PostNuxMlRequest, CandidateUser] .observe(statsReceiver.scope("dedupping")) .andThen( samplingTransform .gated(PostNuxMlParams.SamplingTransformEnabled) .observe(statsReceiver.scope("samplingtransform"))) } override protected val validateCandidates: Predicate[(PostNuxMlRequest, CandidateUser)] = { val stats = statsReceiver.scope("validate_candidates") val competitorPredicate = curatedCompetitorListPredicate.map[(PostNuxMlRequest, CandidateUser)](_._2) val producerHoldbackPredicate = new CandidateParamPredicate[CandidateUser]( GlobalParams.KeepUserCandidate, FilterReason.CandidateSideHoldback ).map[(PostNuxMlRequest, CandidateUser)] { case (request, user) => candidateParamsFactory(user, request) } val pymkProducerHoldbackPredicate = new CandidateSourceParamPredicate( GlobalParams.KeepSocialUserCandidate, FilterReason.CandidateSideHoldback, CandidateSourceHoldbackUtil.SocialCandidateSourceIds ).map[(PostNuxMlRequest, CandidateUser)] { case (request, user) => candidateParamsFactory(user, request) } val sgsPredicateStats = stats.scope("sgs_predicate") object sgsGatedPredicate extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)]( sgsPredicate.observe(sgsPredicateStats), sgsPredicateStats ) { /** * When SGS predicate is turned off, only query SGS exists API for (user, candidate, relationship) * when the user's number of invalid relationships exceeds the threshold during request * building step. This is to minimize load to SGS and underlying Flock DB. */ override def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean = item._1.params(PostNuxMlParams.EnableSGSPredicate) || SocialGraphClient.enablePostRankerSgsPredicate( item._1.invalidRelationshipUserIds.getOrElse(Set.empty).size) } val hssPredicateStats = stats.scope("hss_predicate") object hssGatedPredicate extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)]( hssPredicate.observe(hssPredicateStats), hssPredicateStats ) { override def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean = item._1.params(PostNuxMlParams.EnableHssPredicate) } Predicate .andConcurrently[(PostNuxMlRequest, CandidateUser)]( Seq( competitorPredicate.observe(stats.scope("curated_competitor_predicate")), gizmoduckPredicate.observe(stats.scope("gizmoduck_predicate")), sgsGatedPredicate, hssGatedPredicate, inactivePredicate.observe(stats.scope("inactive_predicate")), ) ) // to avoid dilutions, we need to apply the receiver holdback predicates at the very last step .andThen(pymkProducerHoldbackPredicate.observe(stats.scope("pymk_receiver_side_holdback"))) .andThen(producerHoldbackPredicate.observe(stats.scope("receiver_side_holdback"))) .observe(stats.scope("overall_validate_candidates")) } override protected val transformResults: Transform[PostNuxMlRequest, CandidateUser] = { modifySocialProofTransform .gated(EnableGFSSocialProofTransform) .andThen(trackingTokenTransform) .andThen(randomRankerIdTransform.gated(PostNuxMlParams.LogRandomRankerId)) .andThen(removeAccountProofTransform.gated(PostNuxMlParams.EnableRemoveAccountProofTransform)) } override protected def resultsConfig(request: PostNuxMlRequest): RecommendationResultsConfig = { RecommendationResultsConfig( request.maxResults.getOrElse(request.params(PostNuxMlParams.ResultSizeParam)), request.params(PostNuxMlParams.BatchSizeParam) ) } override def applySideEffects( target: PostNuxMlRequest, candidateSources: Seq[CandidateSource[PostNuxMlRequest, CandidateUser]], candidatesFromCandidateSources: Seq[CandidateUser], mergedCandidates: Seq[CandidateUser], filteredCandidates: Seq[CandidateUser], rankedCandidates: Seq[CandidateUser], transformedCandidates: Seq[CandidateUser], truncatedCandidates: Seq[CandidateUser], results: Seq[CandidateUser] ): Stitch[Unit] = { frsLogger.logRecommendationFlowData[PostNuxMlRequest]( target, RecommendationFlowData[PostNuxMlRequest]( target, PostNuxMlFlow.identifier, candidateSources, candidatesFromCandidateSources, mergedCandidates, filteredCandidates, rankedCandidates, transformedCandidates, truncatedCandidates, results ) ) super.applySideEffects( target, candidateSources, candidatesFromCandidateSources, mergedCandidates, filteredCandidates, rankedCandidates, transformedCandidates, truncatedCandidates, results ) } } object PostNuxMlFlow { val identifier = RecommendationPipelineIdentifier("PostNuxMlFlow") } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeights.scala ================================================ package com.twitter.follow_recommendations.flows.post_nux_ml import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsSource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashQualityFollowSource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonV2Source import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RecentEngagementNonDirectFollowSource import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsSource import com.twitter.follow_recommendations.common.candidate_sources.salsa.RecentEngagementDirectFollowSalsaExpansionSource import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource import com.twitter.follow_recommendations.common.candidate_sources.sims.Follow2vecNearestNeighborsStore import com.twitter.follow_recommendations.common.candidate_sources.stp.BaseOnlineSTPSource import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsSource import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource import com.twitter.follow_recommendations.common.candidate_sources.two_hop_random_walk.TwoHopRandomWalkSource import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphCandidateSource import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlCandidateSourceWeightParams._ import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.timelines.configapi.Params object PostNuxMlFlowCandidateSourceWeights { def getWeights(params: Params): Map[CandidateSourceIdentifier, Double] = { Map[CandidateSourceIdentifier, Double]( // Social based PPMILocaleFollowSource.Identifier -> params(CandidateWeightPPMILocaleFollow), Follow2vecNearestNeighborsStore.IdentifierF2vLinearRegression -> params( CandidateWeightFollow2vecNearestNeighbors), RecentFollowingSimilarUsersSource.Identifier -> params( CandidateWeightRecentFollowingSimilarUsers), BaseOnlineSTPSource.Identifier -> params(CandidateWeightOnlineStp), OfflineStrongTiePredictionSource.Identifier -> params( CandidateWeightOfflineStrongTiePrediction), ForwardEmailBookSource.Identifier -> params(CandidateWeightForwardEmailBook), ForwardPhoneBookSource.Identifier -> params(CandidateWeightForwardPhoneBook), ReverseEmailBookSource.Identifier -> params(CandidateWeightReverseEmailBook), ReversePhoneBookSource.Identifier -> params(CandidateWeightReversePhoneBook), TriangularLoopsSource.Identifier -> params(CandidateWeightTriangularLoops), TwoHopRandomWalkSource.Identifier -> params(CandidateWeightTwoHopRandomWalk), UserUserGraphCandidateSource.Identifier -> params(CandidateWeightUserUserGraph), // Geo based PopCountrySource.Identifier -> params(CandidateWeightPopCountry), PopCountryBackFillSource.Identifier -> params(CandidateWeightPopGeoBackfill), PopGeohashSource.Identifier -> params(CandidateWeightPopGeohash), PopGeohashQualityFollowSource.Identifier -> params(CandidateWeightPopGeohashQualityFollow), CrowdSearchAccountsSource.Identifier -> params(CandidateWeightCrowdSearch), TopOrganicFollowsAccountsSource.Identifier -> params(CandidateWeightTopOrganicFollow), // Engagement based RealGraphOonV2Source.Identifier -> params(CandidateWeightRealGraphOonV2), RecentEngagementNonDirectFollowSource.Identifier -> params( CandidateWeightRecentEngagementNonDirectFollow), RecentEngagementSimilarUsersSource.Identifier -> params( CandidateWeightRecentEngagementSimilarUsers), RepeatedProfileVisitsSource.Identifier -> params(CandidateWeightRepeatedProfileVisits), RecentEngagementDirectFollowSalsaExpansionSource.Identifier -> params( CandidateWeightRecentEngagementDirectFollowSalsaExpansion), ) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.scala ================================================ package com.twitter.follow_recommendations.flows.post_nux_ml object PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys { val CandidateWeightCrowdSearch = "post_nux_ml_flow_candidate_source_weights_user_crowd_search" val CandidateWeightTopOrganicFollow = "post_nux_ml_flow_candidate_source_weights_top_organic_follow" val CandidateWeightPPMILocaleFollow = "post_nux_ml_flow_candidate_source_weights_user_ppmi_locale_follow" val CandidateWeightForwardEmailBook = "post_nux_ml_flow_candidate_source_weights_user_forward_email_book" val CandidateWeightForwardPhoneBook = "post_nux_ml_flow_candidate_source_weights_user_forward_phone_book" val CandidateWeightOfflineStrongTiePrediction = "post_nux_ml_flow_candidate_source_weights_user_offline_strong_tie_prediction" val CandidateWeightOnlineStp = "post_nux_ml_flow_candidate_source_weights_user_online_stp" val CandidateWeightPopCountry = "post_nux_ml_flow_candidate_source_weights_user_pop_country" val CandidateWeightPopGeohash = "post_nux_ml_flow_candidate_source_weights_user_pop_geohash" val CandidateWeightPopGeohashQualityFollow = "post_nux_ml_flow_candidate_source_weights_user_pop_geohash_quality_follow" val CandidateWeightPopGeoBackfill = "post_nux_ml_flow_candidate_source_weights_user_pop_geo_backfill" val CandidateWeightRecentFollowingSimilarUsers = "post_nux_ml_flow_candidate_source_weights_user_recent_following_similar_users" val CandidateWeightRecentEngagementDirectFollowSalsaExpansion = "post_nux_ml_flow_candidate_source_weights_user_recent_engagement_direct_follow_salsa_expansion" val CandidateWeightRecentEngagementNonDirectFollow = "post_nux_ml_flow_candidate_source_weights_user_recent_engagement_non_direct_follow" val CandidateWeightRecentEngagementSimilarUsers = "post_nux_ml_flow_candidate_source_weights_user_recent_engagement_similar_users" val CandidateWeightRepeatedProfileVisits = "post_nux_ml_flow_candidate_source_weights_user_repeated_profile_visits" val CandidateWeightFollow2vecNearestNeighbors = "post_nux_ml_flow_candidate_source_weights_user_follow2vec_nearest_neighbors" val CandidateWeightReverseEmailBook = "post_nux_ml_flow_candidate_source_weights_user_reverse_email_book" val CandidateWeightReversePhoneBook = "post_nux_ml_flow_candidate_source_weights_user_reverse_phone_book" val CandidateWeightTriangularLoops = "post_nux_ml_flow_candidate_source_weights_user_triangular_loops" val CandidateWeightTwoHopRandomWalk = "post_nux_ml_flow_candidate_source_weights_user_two_hop_random_walk" val CandidateWeightUserUserGraph = "post_nux_ml_flow_candidate_source_weights_user_user_user_graph" val CandidateWeightRealGraphOonV2 = "post_nux_ml_flow_candidate_source_weights_user_real_graph_oon_v2" } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFSConfig.scala ================================================ package com.twitter.follow_recommendations.flows.post_nux_ml import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.NoShuffle import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.RandomShuffler import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.timelines.configapi.Param import com.twitter.util.Duration import javax.inject.Inject import javax.inject.Singleton @Singleton class PostNuxMlFlowFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( PostNuxMlParams.OnlineSTPEnabled, PostNuxMlParams.SamplingTransformEnabled, PostNuxMlParams.Follow2VecLinearRegressionEnabled, PostNuxMlParams.UseMlRanker, PostNuxMlParams.EnableCandidateParamHydration, PostNuxMlParams.EnableInterleaveRanker, PostNuxMlParams.EnableAdhocRanker, PostNuxMlParams.ExcludeNearZeroCandidates, PostNuxMlParams.IncludeRepeatedProfileVisitsCandidateSource, PostNuxMlParams.EnableInterestsOptOutPredicate, PostNuxMlParams.EnableSGSPredicate, PostNuxMlParams.EnableInvalidRelationshipPredicate, PostNuxMlParams.EnableRemoveAccountProofTransform, PostNuxMlParams.EnablePPMILocaleFollowSourceInPostNux, PostNuxMlParams.EnableRealGraphOonV2, PostNuxMlParams.GetFollowersFromSgs, PostNuxMlRequestBuilderParams.EnableInvalidRelationshipPredicate ) override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( PostNuxMlCandidateSourceWeightParams.CandidateWeightCrowdSearch, PostNuxMlCandidateSourceWeightParams.CandidateWeightTopOrganicFollow, PostNuxMlCandidateSourceWeightParams.CandidateWeightPPMILocaleFollow, PostNuxMlCandidateSourceWeightParams.CandidateWeightForwardEmailBook, PostNuxMlCandidateSourceWeightParams.CandidateWeightForwardPhoneBook, PostNuxMlCandidateSourceWeightParams.CandidateWeightOfflineStrongTiePrediction, PostNuxMlCandidateSourceWeightParams.CandidateWeightOnlineStp, PostNuxMlCandidateSourceWeightParams.CandidateWeightPopCountry, PostNuxMlCandidateSourceWeightParams.CandidateWeightPopGeohash, PostNuxMlCandidateSourceWeightParams.CandidateWeightPopGeohashQualityFollow, PostNuxMlCandidateSourceWeightParams.CandidateWeightPopGeoBackfill, PostNuxMlCandidateSourceWeightParams.CandidateWeightRecentFollowingSimilarUsers, PostNuxMlCandidateSourceWeightParams.CandidateWeightRecentEngagementDirectFollowSalsaExpansion, PostNuxMlCandidateSourceWeightParams.CandidateWeightRecentEngagementNonDirectFollow, PostNuxMlCandidateSourceWeightParams.CandidateWeightRecentEngagementSimilarUsers, PostNuxMlCandidateSourceWeightParams.CandidateWeightRepeatedProfileVisits, PostNuxMlCandidateSourceWeightParams.CandidateWeightFollow2vecNearestNeighbors, PostNuxMlCandidateSourceWeightParams.CandidateWeightReverseEmailBook, PostNuxMlCandidateSourceWeightParams.CandidateWeightReversePhoneBook, PostNuxMlCandidateSourceWeightParams.CandidateWeightTriangularLoops, PostNuxMlCandidateSourceWeightParams.CandidateWeightTwoHopRandomWalk, PostNuxMlCandidateSourceWeightParams.CandidateWeightUserUserGraph, PostNuxMlCandidateSourceWeightParams.CandidateWeightRealGraphOonV2, PostNuxMlParams.TurnoffMLScorerQFThreshold ) override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( PostNuxMlParams.MlRankerBudget, PostNuxMlRequestBuilderParams.TopicIdFetchBudget, PostNuxMlRequestBuilderParams.DismissedIdScanBudget, PostNuxMlRequestBuilderParams.WTFImpressionsScanBudget ) override val gatedOverridesMap = Map( PostNuxMlFlowFeatureSwitchKeys.EnableRandomDataCollection -> Seq( PostNuxMlParams.CandidateShuffler := new RandomShuffler[CandidateUser], PostNuxMlParams.LogRandomRankerId := true ), PostNuxMlFlowFeatureSwitchKeys.EnableNoShuffler -> Seq( PostNuxMlParams.CandidateShuffler := new NoShuffle[CandidateUser] ), ) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFeatureSwitchKeys.scala ================================================ package com.twitter.follow_recommendations.flows.post_nux_ml object PostNuxMlFlowFeatureSwitchKeys { val UseMlRanker = "post_nux_ml_flow_use_ml_ranker" val EnableCandidateParamHydration = "post_nux_ml_flow_enable_candidate_param_hydration" val OnlineSTPEnabled = "post_nux_ml_flow_online_stp_source_enabled" val Follow2VecLinearRegressionEnabled = "post_nux_ml_flow_follow_to_vec_lr_source_enabled" val EnableRandomDataCollection = "post_nux_ml_flow_random_data_collection_enabled" val EnableAdhocRanker = "post_nux_ml_flow_adhoc_ranker_enabled" val EnableFatigueRanker = "post_nux_ml_flow_fatigue_ranker_enabled" val EnableInterleaveRanker = "post_nux_ml_flow_interleave_ranker_enabled" val IncludeRepeatedProfileVisitsCandidateSource = "post_nux_ml_flow_include_repeated_profile_visits_candidate_source" val MLRankerBudget = "post_nux_ml_flow_ml_ranker_budget_millis" val EnableNoShuffler = "post_nux_ml_flow_no_shuffler" val SamplingTransformEnabled = "post_nux_ml_flow_sampling_transform_enabled" val ExcludeNearZeroCandidates = "post_nux_ml_flow_exclude_near_zero_candidates" val EnableInterestsOptOutPredicate = "post_nux_ml_flow_enable_interests_opt_out_predicate" val EnableRemoveAccountProofTransform = "post_nux_ml_flow_enable_remove_account_proof_transform" val EnablePPMILocaleFollowSourceInPostNux = "post_nux_ml_flow_enable_ppmilocale_follow_source" val EnableInvalidRelationshipPredicate = "post_nux_ml_flow_enable_invalid_relationship_predicate" val EnableRealGraphOonV2 = "post_nux_ml_flow_enable_real_graph_oon_v2" val EnableSGSPredicate = "post_nux_ml_flow_enable_sgs_predicate" val EnableHssPredicate = "post_nux_ml_flow_enable_hss_predicate" val GetFollowersFromSgs = "post_nux_ml_flow_get_followers_from_sgs" val TurnOffMLScorerQFThreshold = "post_nux_ml_flow_turn_off_ml_scorer_threhsold" } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlParams.scala ================================================ package com.twitter.follow_recommendations.flows.post_nux_ml import com.twitter.conversions.DurationOps._ import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.CandidateShuffler import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.ExponentialShuffler import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.timelines.configapi.Param import com.twitter.util.Duration abstract class PostNuxMlParams[A](default: A) extends Param[A](default) { override val statName: String = "post_nux_ml/" + this.getClass.getSimpleName } object PostNuxMlParams { // infra params: case object FetchCandidateSourceBudget extends PostNuxMlParams[Duration](90.millisecond) // WTF Impression Store has very high tail latency (p9990 or p9999), but p99 latency is pretty good (~100ms) // set the time budget for this step to be 200ms to make the performance of service more predictable case object FatigueRankerBudget extends PostNuxMlParams[Duration](200.millisecond) case object MlRankerBudget extends FSBoundedParam[Duration]( name = PostNuxMlFlowFeatureSwitchKeys.MLRankerBudget, default = 400.millisecond, min = 100.millisecond, max = 800.millisecond) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMillis } // product params: case object TargetEligibility extends PostNuxMlParams[Boolean](true) case object ResultSizeParam extends PostNuxMlParams[Int](3) case object BatchSizeParam extends PostNuxMlParams[Int](12) case object CandidateShuffler extends PostNuxMlParams[CandidateShuffler[CandidateUser]]( new ExponentialShuffler[CandidateUser]) case object LogRandomRankerId extends PostNuxMlParams[Boolean](false) // whether or not to use the ml ranker at all (feature hydration + ranker) case object UseMlRanker extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.UseMlRanker, false) // whether or not to enable candidate param hydration in postnux_ml_flow case object EnableCandidateParamHydration extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableCandidateParamHydration, false) // Whether or not OnlineSTP candidates are considered in the final pool of candidates. // If set to `false`, the candidate source will be removed *after* all other considerations. case object OnlineSTPEnabled extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.OnlineSTPEnabled, false) // Whether or not the candidates are sampled from a Plackett-Luce model case object SamplingTransformEnabled extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.SamplingTransformEnabled, false) // Whether or not Follow2Vec candidates are considered in the final pool of candidates. // If set to `false`, the candidate source will be removed *after* all other considerations. case object Follow2VecLinearRegressionEnabled extends FSParam[Boolean]( PostNuxMlFlowFeatureSwitchKeys.Follow2VecLinearRegressionEnabled, false) // Whether or not to enable AdhocRanker to allow adhoc, non-ML, score modifications. case object EnableAdhocRanker extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableAdhocRanker, false) // Whether the impression-based fatigue ranker is enabled or not. case object EnableFatigueRanker extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableFatigueRanker, true) // whether or not to enable InterleaveRanker for producer-side experiments. case object EnableInterleaveRanker extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableInterleaveRanker, false) // whether to exclude users in near zero user state case object ExcludeNearZeroCandidates extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.ExcludeNearZeroCandidates, false) case object EnablePPMILocaleFollowSourceInPostNux extends FSParam[Boolean]( PostNuxMlFlowFeatureSwitchKeys.EnablePPMILocaleFollowSourceInPostNux, false) case object EnableInterestsOptOutPredicate extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableInterestsOptOutPredicate, false) case object EnableInvalidRelationshipPredicate extends FSParam[Boolean]( PostNuxMlFlowFeatureSwitchKeys.EnableInvalidRelationshipPredicate, false) // Totally disabling SGS predicate need to disable EnableInvalidRelationshipPredicate as well case object EnableSGSPredicate extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableSGSPredicate, true) case object EnableHssPredicate extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableHssPredicate, true) // Whether or not to include RepeatedProfileVisits as one of the candidate sources in the PostNuxMlFlow. If false, // RepeatedProfileVisitsSource would not be run for the users in candidate_generation. case object IncludeRepeatedProfileVisitsCandidateSource extends FSParam[Boolean]( PostNuxMlFlowFeatureSwitchKeys.IncludeRepeatedProfileVisitsCandidateSource, false) case object EnableRealGraphOonV2 extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableRealGraphOonV2, false) case object GetFollowersFromSgs extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.GetFollowersFromSgs, false) case object EnableRemoveAccountProofTransform extends FSParam[Boolean]( PostNuxMlFlowFeatureSwitchKeys.EnableRemoveAccountProofTransform, false) // quality factor threshold to turn off ML ranker completely object TurnoffMLScorerQFThreshold extends FSBoundedParam[Double]( name = PostNuxMlFlowFeatureSwitchKeys.TurnOffMLScorerQFThreshold, default = 0.3, min = 0.1, max = 1.0) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequest.scala ================================================ package com.twitter.follow_recommendations.flows.post_nux_ml import com.twitter.core_workflows.user_model.thriftscala.UserState import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature import com.twitter.follow_recommendations.common.models._ import com.twitter.product_mixer.core.model.marshalling.request.ClientContext import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.timelines.configapi.HasParams import com.twitter.timelines.configapi.Params case class PostNuxMlRequest( override val params: Params, override val clientContext: ClientContext, override val similarToUserIds: Seq[Long], inputExcludeUserIds: Seq[Long], override val recentFollowedUserIds: Option[Seq[Long]], override val invalidRelationshipUserIds: Option[Set[Long]], override val recentFollowedByUserIds: Option[Seq[Long]], override val dismissedUserIds: Option[Seq[Long]], override val displayLocation: DisplayLocation, maxResults: Option[Int] = None, override val debugOptions: Option[DebugOptions] = None, override val wtfImpressions: Option[Seq[WtfImpression]], override val uttInterestIds: Option[Seq[Long]] = None, override val customInterests: Option[Seq[String]] = None, override val geohashAndCountryCode: Option[GeohashAndCountryCode] = None, inputPreviouslyRecommendedUserIds: Option[Set[Long]] = None, inputPreviouslyFollowedUserIds: Option[Set[Long]] = None, override val isSoftUser: Boolean = false, override val userState: Option[UserState] = None, override val qualityFactor: Option[Double] = None) extends HasParams with HasSimilarToContext with HasClientContext with HasExcludedUserIds with HasDisplayLocation with HasDebugOptions with HasGeohashAndCountryCode with HasPreFetchedFeature with HasDismissedUserIds with HasInterestIds with HasPreviousRecommendationsContext with HasIsSoftUser with HasUserState with HasInvalidRelationshipUserIds with HasQualityFactor { override val excludedUserIds: Seq[Long] = { inputExcludeUserIds ++ clientContext.userId.toSeq ++ similarToUserIds } override val previouslyRecommendedUserIDs: Set[Long] = inputPreviouslyRecommendedUserIds.getOrElse(Set.empty) override val previouslyFollowedUserIds: Set[Long] = inputPreviouslyFollowedUserIds.getOrElse(Set.empty) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilder.scala ================================================ package com.twitter.follow_recommendations.flows.post_nux_ml import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.clients.dismiss_store.DismissStore import com.twitter.follow_recommendations.common.clients.geoduck.UserLocationFetcher import com.twitter.follow_recommendations.common.clients.impression_store.WtfImpressionStore import com.twitter.follow_recommendations.common.clients.interests_service.InterestServiceClient import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient import com.twitter.follow_recommendations.common.clients.user_state.UserStateClient import com.twitter.follow_recommendations.common.predicates.dismiss.DismissedCandidatePredicateParams import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils._ import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilderParams.DismissedIdScanBudget import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilderParams.TopicIdFetchBudget import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilderParams.WTFImpressionsScanBudget import com.twitter.follow_recommendations.products.common.ProductRequest import com.twitter.inject.Logging import com.twitter.stitch.Stitch import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton @Singleton class PostNuxMlRequestBuilder @Inject() ( socialGraph: SocialGraphClient, wtfImpressionStore: WtfImpressionStore, dismissStore: DismissStore, userLocationFetcher: UserLocationFetcher, interestServiceClient: InterestServiceClient, userStateClient: UserStateClient, statsReceiver: StatsReceiver) extends Logging { val stats: StatsReceiver = statsReceiver.scope("post_nux_ml_request_builder") val invalidRelationshipUsersStats: StatsReceiver = stats.scope("invalidRelationshipUserIds") private val invalidRelationshipUsersMaxSizeCounter = invalidRelationshipUsersStats.counter("maxSize") private val invalidRelationshipUsersNotMaxSizeCounter = invalidRelationshipUsersStats.counter("notMaxSize") def build( req: ProductRequest, previouslyRecommendedUserIds: Option[Set[Long]] = None, previouslyFollowedUserIds: Option[Set[Long]] = None ): Stitch[PostNuxMlRequest] = { val dl = req.recommendationRequest.displayLocation val resultsStitch = Stitch.collect( req.recommendationRequest.clientContext.userId .map { userId => val lookBackDuration = req.params(DismissedCandidatePredicateParams.LookBackDuration) val negativeStartTs = -(Time.now - lookBackDuration).inMillis val recentFollowedUserIdsStitch = rescueWithStats( socialGraph.getRecentFollowedUserIds(userId), stats, "recentFollowedUserIds") val invalidRelationshipUserIdsStitch = if (req.params(PostNuxMlParams.EnableInvalidRelationshipPredicate)) { rescueWithStats( socialGraph .getInvalidRelationshipUserIds(userId) .onSuccess(ids => if (ids.size >= SocialGraphClient.MaxNumInvalidRelationship) { invalidRelationshipUsersMaxSizeCounter.incr() } else { invalidRelationshipUsersNotMaxSizeCounter.incr() }), stats, "invalidRelationshipUserIds" ) } else { Stitch.value(Seq.empty) } // recentFollowedByUserIds are only used in experiment candidate sources val recentFollowedByUserIdsStitch = if (req.params(PostNuxMlParams.GetFollowersFromSgs)) { rescueWithStats( socialGraph.getRecentFollowedByUserIdsFromCachedColumn(userId), stats, "recentFollowedByUserIds") } else Stitch.value(Seq.empty) val wtfImpressionsStitch = rescueWithStatsWithin( wtfImpressionStore.get(userId, dl), stats, "wtfImpressions", req.params(WTFImpressionsScanBudget)) val dismissedUserIdsStitch = rescueWithStatsWithin( dismissStore.get(userId, negativeStartTs, None), stats, "dismissedUserIds", req.params(DismissedIdScanBudget)) val locationStitch = rescueOptionalWithStats( userLocationFetcher.getGeohashAndCountryCode( Some(userId), req.recommendationRequest.clientContext.ipAddress), stats, "userLocation" ) val topicIdsStitch = rescueWithStatsWithin( interestServiceClient.fetchUttInterestIds(userId), stats, "topicIds", req.params(TopicIdFetchBudget)) val userStateStitch = rescueOptionalWithStats(userStateClient.getUserState(userId), stats, "userState") Stitch.join( recentFollowedUserIdsStitch, invalidRelationshipUserIdsStitch, recentFollowedByUserIdsStitch, dismissedUserIdsStitch, wtfImpressionsStitch, locationStitch, topicIdsStitch, userStateStitch ) }) resultsStitch.map { case Some( ( recentFollowedUserIds, invalidRelationshipUserIds, recentFollowedByUserIds, dismissedUserIds, wtfImpressions, locationInfo, topicIds, userState)) => PostNuxMlRequest( params = req.params, clientContext = req.recommendationRequest.clientContext, similarToUserIds = Nil, inputExcludeUserIds = req.recommendationRequest.excludedIds.getOrElse(Nil), recentFollowedUserIds = Some(recentFollowedUserIds), invalidRelationshipUserIds = Some(invalidRelationshipUserIds.toSet), recentFollowedByUserIds = Some(recentFollowedByUserIds), dismissedUserIds = Some(dismissedUserIds), displayLocation = dl, maxResults = req.recommendationRequest.maxResults, debugOptions = req.recommendationRequest.debugParams.flatMap(_.debugOptions), wtfImpressions = Some(wtfImpressions), geohashAndCountryCode = locationInfo, uttInterestIds = Some(topicIds), inputPreviouslyRecommendedUserIds = previouslyRecommendedUserIds, inputPreviouslyFollowedUserIds = previouslyFollowedUserIds, isSoftUser = req.recommendationRequest.isSoftUser, userState = userState ) case _ => PostNuxMlRequest( params = req.params, clientContext = req.recommendationRequest.clientContext, similarToUserIds = Nil, inputExcludeUserIds = req.recommendationRequest.excludedIds.getOrElse(Nil), recentFollowedUserIds = None, invalidRelationshipUserIds = None, recentFollowedByUserIds = None, dismissedUserIds = None, displayLocation = dl, maxResults = req.recommendationRequest.maxResults, debugOptions = req.recommendationRequest.debugParams.flatMap(_.debugOptions), wtfImpressions = None, geohashAndCountryCode = None, inputPreviouslyRecommendedUserIds = previouslyRecommendedUserIds, inputPreviouslyFollowedUserIds = previouslyFollowedUserIds, isSoftUser = req.recommendationRequest.isSoftUser, userState = None ) } } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilderParams.scala ================================================ package com.twitter.follow_recommendations.flows.post_nux_ml import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.util.Duration import com.twitter.conversions.DurationOps._ import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.HasDurationConversion object PostNuxMlRequestBuilderParams { case object TopicIdFetchBudget extends FSBoundedParam[Duration]( name = "post_nux_ml_request_builder_topic_id_fetch_budget_millis", default = 200.millisecond, min = 80.millisecond, max = 400.millisecond) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMillis } case object DismissedIdScanBudget extends FSBoundedParam[Duration]( name = "post_nux_ml_request_builder_dismissed_id_scan_budget_millis", default = 200.millisecond, min = 80.millisecond, max = 400.millisecond) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMillis } case object WTFImpressionsScanBudget extends FSBoundedParam[Duration]( name = "post_nux_ml_request_builder_wtf_impressions_scan_budget_millis", default = 200.millisecond, min = 80.millisecond, max = 400.millisecond) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMillis } case object EnableInvalidRelationshipPredicate extends FSParam[Boolean]( name = "post_nux_ml_request_builder_enable_invalid_relationship_predicate", false) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "scribelib/marshallers/src/main/scala/com/twitter/scribelib/marshallers", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/FrsLogger.scala ================================================ package com.twitter.follow_recommendations.logging import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.follow_recommendations.common.models.HasIsSoftUser import com.twitter.follow_recommendations.configapi.params.GlobalParams import com.twitter.follow_recommendations.logging.thriftscala.RecommendationLog import com.twitter.follow_recommendations.models.DebugParams import com.twitter.follow_recommendations.models.RecommendationFlowData import com.twitter.follow_recommendations.models.RecommendationRequest import com.twitter.follow_recommendations.models.RecommendationResponse import com.twitter.follow_recommendations.models.ScoringUserRequest import com.twitter.follow_recommendations.models.ScoringUserResponse import com.twitter.inject.annotations.Flag import com.twitter.logging.LoggerFactory import com.twitter.product_mixer.core.model.marshalling.request.ClientContext import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.scribelib.marshallers.ClientDataProvider import com.twitter.scribelib.marshallers.ExternalRefererDataProvider import com.twitter.scribelib.marshallers.ScribeSerialization import com.twitter.timelines.configapi.HasParams import com.twitter.util.Time import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton /** * This is the standard logging class we use to log data into: * 1) logs.follow_recommendations_logs * * This logger logs data for 2 endpoints: getRecommendations, scoreUserCandidates * All data scribed via this logger have to be converted into the same thrift type: RecommendationLog * * 2) logs.frs_recommendation_flow_logs * * This logger logs recommendation flow data for getRecommendations requests * All data scribed via this logger have to be converted into the same thrift type: FrsRecommendationFlowLog */ @Singleton class FrsLogger @Inject() ( @Named(GuiceNamedConstants.REQUEST_LOGGER) loggerFactory: LoggerFactory, @Named(GuiceNamedConstants.FLOW_LOGGER) flowLoggerFactory: LoggerFactory, stats: StatsReceiver, @Flag("log_results") serviceShouldLogResults: Boolean) extends ScribeSerialization { private val logger = loggerFactory.apply() private val flowLogger = flowLoggerFactory.apply() private val logRecommendationCounter = stats.counter("scribe_recommendation") private val logScoringCounter = stats.counter("scribe_scoring") private val logRecommendationFlowCounter = stats.counter("scribe_recommendation_flow") def logRecommendationResult( request: RecommendationRequest, response: RecommendationResponse ): Unit = { if (!request.isSoftUser) { val log = RecommendationLog(request.toOfflineThrift, response.toOfflineThrift, Time.now.inMillis) logRecommendationCounter.incr() logger.info( serializeThrift( log, FrsLogger.LogCategory, FrsLogger.mkProvider(request.clientContext) )) } } def logScoringResult(request: ScoringUserRequest, response: ScoringUserResponse): Unit = { if (!request.isSoftUser) { val log = RecommendationLog( request.toRecommendationRequest.toOfflineThrift, response.toRecommendationResponse.toOfflineThrift, Time.now.inMillis) logScoringCounter.incr() logger.info( serializeThrift( log, FrsLogger.LogCategory, FrsLogger.mkProvider(request.toRecommendationRequest.clientContext) )) } } def logRecommendationFlowData[Target <: HasClientContext with HasIsSoftUser with HasParams]( request: Target, flowData: RecommendationFlowData[Target] ): Unit = { if (!request.isSoftUser && request.params(GlobalParams.EnableRecommendationFlowLogs)) { val log = flowData.toRecommendationFlowLogOfflineThrift logRecommendationFlowCounter.incr() flowLogger.info( serializeThrift( log, FrsLogger.FlowLogCategory, FrsLogger.mkProvider(request.clientContext) )) } } // We prefer the settings given in the user request, and if none provided we default to the // aurora service configuration. def shouldLog(debugParamsOpt: Option[DebugParams]): Boolean = debugParamsOpt match { case Some(debugParams) => debugParams.debugOptions match { case Some(debugOptions) => !debugOptions.doNotLog case None => serviceShouldLogResults } case None => serviceShouldLogResults } } object FrsLogger { val LogCategory = "follow_recommendations_logs" val FlowLogCategory = "frs_recommendation_flow_logs" def mkProvider(clientContext: ClientContext) = new ClientDataProvider { /** The id of the current user. When the user is logged out, this method should return None. */ override val userId: Option[Long] = clientContext.userId /** The id of the guest, which is present in logged-in or loged-out states */ override val guestId: Option[Long] = clientContext.guestId /** The personalization id (pid) of the user, used to personalize Twitter services */ override val personalizationId: Option[String] = None /** The id of the individual device the user is currently using. This id will be unique for different users' devices. */ override val deviceId: Option[String] = clientContext.deviceId /** The OAuth application id of the application the user is currently using */ override val clientApplicationId: Option[Long] = clientContext.appId /** The OAuth parent application id of the application the user is currently using */ override val parentApplicationId: Option[Long] = None /** The two-letter, upper-case country code used to designate the country from which the scribe event occurred */ override val countryCode: Option[String] = clientContext.countryCode /** The two-letter, lower-case language code used to designate the probably language spoken by the scribe event initiator */ override val languageCode: Option[String] = clientContext.languageCode /** The user-agent header used to identify the client browser or device that the user is currently active on */ override val userAgent: Option[String] = clientContext.userAgent /** Whether the user is accessing Twitter via a secured connection */ override val isSsl: Option[Boolean] = Some(true) /** The referring URL to the current page for web-based clients, if applicable */ override val referer: Option[String] = None /** * The external site, partner, or email that lead to the current Twitter application. Returned value consists of a * tuple including the encrypted referral data and the type of referral */ override val externalReferer: Option[ExternalRefererDataProvider] = None } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models", "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateSourceType.scala ================================================ package com.twitter.follow_recommendations.models object CandidateSourceType extends Enumeration { type CandidateSourceType = Value val Social = Value("social") val GeoAndInterests = Value("geo_and_interests") val ActivityContextual = Value("activity_contextual") val None = Value("none") } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateUserDebugParams.scala ================================================ package com.twitter.follow_recommendations.models import com.twitter.timelines.configapi.Params case class CandidateUserDebugParams(paramsMap: Map[Long, Params]) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DebugParams.scala ================================================ package com.twitter.follow_recommendations.models import com.twitter.follow_recommendations.common.models.DebugOptions import com.twitter.follow_recommendations.common.models.DebugOptions.fromDebugParamsThrift import com.twitter.follow_recommendations.logging.{thriftscala => offline} import com.twitter.follow_recommendations.{thriftscala => t} import com.twitter.timelines.configapi.{FeatureValue => ConfigApiFeatureValue} case class DebugParams( featureOverrides: Option[Map[String, ConfigApiFeatureValue]], debugOptions: Option[DebugOptions]) object DebugParams { def fromThrift(thrift: t.DebugParams): DebugParams = DebugParams( featureOverrides = thrift.featureOverrides.map { map => map.mapValues(FeatureValue.fromThrift).toMap }, debugOptions = Some( fromDebugParamsThrift(thrift) ) ) def toOfflineThrift(model: DebugParams): offline.OfflineDebugParams = offline.OfflineDebugParams(randomizationSeed = model.debugOptions.flatMap(_.randomizationSeed)) } trait HasFrsDebugParams { def frsDebugParams: Option[DebugParams] } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DisplayContext.scala ================================================ package com.twitter.follow_recommendations.models import com.twitter.follow_recommendations.common.models.FlowContext import com.twitter.follow_recommendations.common.models.RecentlyEngagedUserId import com.twitter.follow_recommendations.logging.thriftscala.OfflineDisplayContext import com.twitter.follow_recommendations.logging.{thriftscala => offline} import com.twitter.follow_recommendations.{thriftscala => t} import scala.reflect.ClassTag import scala.reflect.classTag trait DisplayContext { def toOfflineThrift: offline.OfflineDisplayContext } object DisplayContext { case class Profile(profileId: Long) extends DisplayContext { override val toOfflineThrift: OfflineDisplayContext = offline.OfflineDisplayContext.Profile(offline.OfflineProfile(profileId)) } case class Search(searchQuery: String) extends DisplayContext { override val toOfflineThrift: OfflineDisplayContext = offline.OfflineDisplayContext.Search(offline.OfflineSearch(searchQuery)) } case class Rux(focalAuthorId: Long) extends DisplayContext { override val toOfflineThrift: OfflineDisplayContext = offline.OfflineDisplayContext.Rux(offline.OfflineRux(focalAuthorId)) } case class Topic(topicId: Long) extends DisplayContext { override val toOfflineThrift: OfflineDisplayContext = offline.OfflineDisplayContext.Topic(offline.OfflineTopic(topicId)) } case class ReactiveFollow(followedUserIds: Seq[Long]) extends DisplayContext { override val toOfflineThrift: OfflineDisplayContext = offline.OfflineDisplayContext.ReactiveFollow(offline.OfflineReactiveFollow(followedUserIds)) } case class NuxInterests(flowContext: Option[FlowContext], uttInterestIds: Option[Seq[Long]]) extends DisplayContext { override val toOfflineThrift: OfflineDisplayContext = offline.OfflineDisplayContext.NuxInterests( offline.OfflineNuxInterests(flowContext.map(_.toOfflineThrift))) } case class PostNuxFollowTask(flowContext: Option[FlowContext]) extends DisplayContext { override val toOfflineThrift: OfflineDisplayContext = offline.OfflineDisplayContext.PostNuxFollowTask( offline.OfflinePostNuxFollowTask(flowContext.map(_.toOfflineThrift))) } case class AdCampaignTarget(similarToUserIds: Seq[Long]) extends DisplayContext { override val toOfflineThrift: OfflineDisplayContext = offline.OfflineDisplayContext.AdCampaignTarget( offline.OfflineAdCampaignTarget(similarToUserIds)) } case class ConnectTab( byfSeedUserIds: Seq[Long], similarToUserIds: Seq[Long], engagedUserIds: Seq[RecentlyEngagedUserId]) extends DisplayContext { override val toOfflineThrift: OfflineDisplayContext = offline.OfflineDisplayContext.ConnectTab( offline.OfflineConnectTab( byfSeedUserIds, similarToUserIds, engagedUserIds.map(user => user.toOfflineThrift))) } case class SimilarToUser(similarToUserId: Long) extends DisplayContext { override val toOfflineThrift: OfflineDisplayContext = offline.OfflineDisplayContext.SimilarToUser(offline.OfflineSimilarToUser(similarToUserId)) } def fromThrift(tDisplayContext: t.DisplayContext): DisplayContext = tDisplayContext match { case t.DisplayContext.Profile(p) => Profile(p.profileId) case t.DisplayContext.Search(s) => Search(s.searchQuery) case t.DisplayContext.Rux(r) => Rux(r.focalAuthorId) case t.DisplayContext.Topic(t) => Topic(t.topicId) case t.DisplayContext.ReactiveFollow(f) => ReactiveFollow(f.followedUserIds) case t.DisplayContext.NuxInterests(n) => NuxInterests(n.flowContext.map(FlowContext.fromThrift), n.uttInterestIds) case t.DisplayContext.AdCampaignTarget(a) => AdCampaignTarget(a.similarToUserIds) case t.DisplayContext.ConnectTab(connect) => ConnectTab( connect.byfSeedUserIds, connect.similarToUserIds, connect.recentlyEngagedUserIds.map(RecentlyEngagedUserId.fromThrift)) case t.DisplayContext.SimilarToUser(r) => SimilarToUser(r.similarToUserId) case t.DisplayContext.PostNuxFollowTask(p) => PostNuxFollowTask(p.flowContext.map(FlowContext.fromThrift)) case t.DisplayContext.UnknownUnionField(t) => throw new UnknownDisplayContextException(t.field.name) } def getDisplayContextAs[T <: DisplayContext: ClassTag](displayContext: DisplayContext): T = displayContext match { case context: T => context case _ => throw new UnexpectedDisplayContextTypeException( displayContext, classTag[T].getClass.getSimpleName) } } class UnknownDisplayContextException(name: String) extends Exception(s"Unknown DisplayContext in Thrift: ${name}") class UnexpectedDisplayContextTypeException(displayContext: DisplayContext, expectedType: String) extends Exception(s"DisplayContext ${displayContext} not of expected type ${expectedType}") ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/FeatureValue.scala ================================================ package com.twitter.follow_recommendations.models import com.twitter.follow_recommendations.{thriftscala => t} import com.twitter.timelines.configapi._ object FeatureValue { def fromThrift(thriftFeatureValue: t.FeatureValue): FeatureValue = thriftFeatureValue match { case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.BoolValue(bool)) => BooleanFeatureValue(bool) case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.StrValue(string)) => StringFeatureValue(string) case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.IntValue(int)) => NumberFeatureValue(int) case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.LongValue(long)) => NumberFeatureValue(long) case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.UnknownUnionField(field)) => throw new UnknownFeatureValueException(s"Primitive: ${field.field.name}") case t.FeatureValue.UnknownUnionField(field) => throw new UnknownFeatureValueException(field.field.name) } } class UnknownFeatureValueException(fieldName: String) extends Exception(s"Unknown FeatureValue name in thrift: ${fieldName}") ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationFlowData.scala ================================================ package com.twitter.follow_recommendations.models import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.ClientContextConverter import com.twitter.follow_recommendations.common.models.HasUserState import com.twitter.follow_recommendations.common.utils.UserSignupUtil import com.twitter.follow_recommendations.logging.{thriftscala => offline} import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.HasMarshalling import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.util.Time case class RecommendationFlowData[Target <: HasClientContext]( request: Target, recommendationFlowIdentifier: RecommendationPipelineIdentifier, candidateSources: Seq[CandidateSource[Target, CandidateUser]], candidatesFromCandidateSources: Seq[CandidateUser], mergedCandidates: Seq[CandidateUser], filteredCandidates: Seq[CandidateUser], rankedCandidates: Seq[CandidateUser], transformedCandidates: Seq[CandidateUser], truncatedCandidates: Seq[CandidateUser], results: Seq[CandidateUser]) extends HasMarshalling { import RecommendationFlowData._ lazy val toRecommendationFlowLogOfflineThrift: offline.RecommendationFlowLog = { val userMetadata = userToOfflineRecommendationFlowUserMetadata(request) val signals = userToOfflineRecommendationFlowSignals(request) val filteredCandidateSourceCandidates = candidatesToOfflineRecommendationFlowCandidateSourceCandidates( candidateSources, filteredCandidates ) val rankedCandidateSourceCandidates = candidatesToOfflineRecommendationFlowCandidateSourceCandidates( candidateSources, rankedCandidates ) val truncatedCandidateSourceCandidates = candidatesToOfflineRecommendationFlowCandidateSourceCandidates( candidateSources, truncatedCandidates ) offline.RecommendationFlowLog( ClientContextConverter.toFRSOfflineClientContextThrift(request.clientContext), userMetadata, signals, Time.now.inMillis, recommendationFlowIdentifier.name, Some(filteredCandidateSourceCandidates), Some(rankedCandidateSourceCandidates), Some(truncatedCandidateSourceCandidates) ) } } object RecommendationFlowData { def userToOfflineRecommendationFlowUserMetadata[Target <: HasClientContext]( request: Target ): Option[offline.OfflineRecommendationFlowUserMetadata] = { val userSignupAge = UserSignupUtil.userSignupAge(request).map(_.inDays) val userState = request match { case req: HasUserState => req.userState.map(_.name) case _ => None } Some(offline.OfflineRecommendationFlowUserMetadata(userSignupAge, userState)) } def userToOfflineRecommendationFlowSignals[Target <: HasClientContext]( request: Target ): Option[offline.OfflineRecommendationFlowSignals] = { val countryCode = request.getCountryCode Some(offline.OfflineRecommendationFlowSignals(countryCode)) } def candidatesToOfflineRecommendationFlowCandidateSourceCandidates[Target <: HasClientContext]( candidateSources: Seq[CandidateSource[Target, CandidateUser]], candidates: Seq[CandidateUser], ): Seq[offline.OfflineRecommendationFlowCandidateSourceCandidates] = { val candidatesGroupedByCandidateSources = candidates.groupBy( _.getPrimaryCandidateSource.getOrElse(CandidateSourceIdentifier("NoCandidateSource"))) candidateSources.map(candidateSource => { val candidates = candidatesGroupedByCandidateSources.get(candidateSource.identifier).toSeq.flatten val candidateUserIds = candidates.map(_.id) val candidateUserScores = candidates.map(_.score).exists(_.nonEmpty) match { case true => Some(candidates.map(_.score.getOrElse(-1.0))) case false => None } offline.OfflineRecommendationFlowCandidateSourceCandidates( candidateSource.identifier.name, candidateUserIds, candidateUserScores ) }) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationRequest.scala ================================================ package com.twitter.follow_recommendations.models import com.twitter.follow_recommendations.common.models.ClientContextConverter import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.follow_recommendations.logging.{thriftscala => offline} import com.twitter.product_mixer.core.model.marshalling.request.ClientContext case class RecommendationRequest( clientContext: ClientContext, displayLocation: DisplayLocation, displayContext: Option[DisplayContext], maxResults: Option[Int], cursor: Option[String], excludedIds: Option[Seq[Long]], fetchPromotedContent: Option[Boolean], debugParams: Option[DebugParams] = None, userLocationState: Option[String] = None, isSoftUser: Boolean = false) { def toOfflineThrift: offline.OfflineRecommendationRequest = offline.OfflineRecommendationRequest( ClientContextConverter.toFRSOfflineClientContextThrift(clientContext), displayLocation.toOfflineThrift, displayContext.map(_.toOfflineThrift), maxResults, cursor, excludedIds, fetchPromotedContent, debugParams.map(DebugParams.toOfflineThrift) ) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationResponse.scala ================================================ package com.twitter.follow_recommendations.models import com.twitter.follow_recommendations.{thriftscala => t} import com.twitter.follow_recommendations.logging.{thriftscala => offline} import com.twitter.follow_recommendations.common.models.Recommendation import com.twitter.product_mixer.core.model.marshalling.HasMarshalling case class RecommendationResponse(recommendations: Seq[Recommendation]) extends HasMarshalling { lazy val toThrift: t.RecommendationResponse = t.RecommendationResponse(recommendations.map(_.toThrift)) lazy val toOfflineThrift: offline.OfflineRecommendationResponse = offline.OfflineRecommendationResponse(recommendations.map(_.toOfflineThrift)) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/Request.scala ================================================ package com.twitter.follow_recommendations.models import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.product_mixer.core.model.marshalling.request import com.twitter.product_mixer.core.model.marshalling.request.ClientContext import com.twitter.product_mixer.core.model.marshalling.request.ProductContext import com.twitter.product_mixer.core.model.marshalling.request.{Request => ProductMixerRequest} case class Request( override val maxResults: Option[Int], override val debugParams: Option[request.DebugParams], override val productContext: Option[ProductContext], override val product: request.Product, override val clientContext: ClientContext, override val serializedRequestCursor: Option[String], override val frsDebugParams: Option[DebugParams], displayLocation: DisplayLocation, excludedIds: Option[Seq[Long]], fetchPromotedContent: Option[Boolean], userLocationState: Option[String] = None) extends ProductMixerRequest with HasFrsDebugParams {} ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserRequest.scala ================================================ package com.twitter.follow_recommendations.models import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature import com.twitter.follow_recommendations.common.models._ import com.twitter.follow_recommendations.logging.{thriftscala => offline} import com.twitter.product_mixer.core.model.marshalling.request.ClientContext import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.timelines.configapi.HasParams import com.twitter.timelines.configapi.Params case class ScoringUserRequest( override val clientContext: ClientContext, override val displayLocation: DisplayLocation, override val params: Params, override val debugOptions: Option[DebugOptions] = None, override val recentFollowedUserIds: Option[Seq[Long]], override val recentFollowedByUserIds: Option[Seq[Long]], override val wtfImpressions: Option[Seq[WtfImpression]], override val similarToUserIds: Seq[Long], candidates: Seq[CandidateUser], debugParams: Option[DebugParams] = None, isSoftUser: Boolean = false) extends HasClientContext with HasDisplayLocation with HasParams with HasDebugOptions with HasPreFetchedFeature with HasSimilarToContext { def toOfflineThrift: offline.OfflineScoringUserRequest = offline.OfflineScoringUserRequest( ClientContextConverter.toFRSOfflineClientContextThrift(clientContext), displayLocation.toOfflineThrift, candidates.map(_.toOfflineUserThrift) ) def toRecommendationRequest: RecommendationRequest = RecommendationRequest( clientContext = clientContext, displayLocation = displayLocation, displayContext = None, maxResults = None, cursor = None, excludedIds = None, fetchPromotedContent = None, debugParams = debugParams, isSoftUser = isSoftUser ) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserResponse.scala ================================================ package com.twitter.follow_recommendations.models import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.logging.{thriftscala => offline} import com.twitter.follow_recommendations.{thriftscala => t} case class ScoringUserResponse(candidates: Seq[CandidateUser]) { lazy val toThrift: t.ScoringUserResponse = t.ScoringUserResponse(candidates.map(_.toUserThrift)) lazy val toRecommendationResponse: RecommendationResponse = RecommendationResponse(candidates) lazy val toOfflineThrift: offline.OfflineScoringUserResponse = offline.OfflineScoringUserResponse(candidates.map(_.toOfflineUserThrift)) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/TimeoutPipelineFailure.scala ================================================ package com.twitter.follow_recommendations.models.failures import com.twitter.product_mixer.core.pipeline.pipeline_failure.CandidateSourceTimeout import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure object TimeoutPipelineFailure { def apply(candidateSourceName: String): PipelineFailure = { PipelineFailure( CandidateSourceTimeout, s"Candidate Source $candidateSourceName timed out before returning candidates") } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ABDeciderModule.scala ================================================ package com.twitter.follow_recommendations.modules import com.google.inject.Provides import com.google.inject.name.Named import com.twitter.abdecider.ABDeciderFactory import com.twitter.abdecider.LoggingABDecider import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.inject.TwitterModule import com.twitter.logging.LoggerFactory import javax.inject.Singleton object ABDeciderModule extends TwitterModule { @Provides @Singleton def provideABDecider( stats: StatsReceiver, @Named(GuiceNamedConstants.CLIENT_EVENT_LOGGER) factory: LoggerFactory ): LoggingABDecider = { val ymlPath = "/usr/local/config/abdecider/abdecider.yml" val abDeciderFactory = ABDeciderFactory( abDeciderYmlPath = ymlPath, scribeLogger = Some(factory()), environment = Some("production") ) abDeciderFactory.buildWithLogging() } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/javax/inject:javax.inject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/slf4j:slf4j-api", "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products", "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", "twml/runtime/src/main/scala/com/twitter/deepbird/runtime/prediction_engine", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ConfigApiModule.scala ================================================ package com.twitter.follow_recommendations.modules import com.google.inject.Provides import com.twitter.decider.Decider import com.twitter.follow_recommendations.configapi.ConfigBuilder import com.twitter.inject.TwitterModule import com.twitter.servo.decider.DeciderGateBuilder import com.twitter.timelines.configapi.Config import javax.inject.Singleton object ConfigApiModule extends TwitterModule { @Provides @Singleton def providesDeciderGateBuilder(decider: Decider): DeciderGateBuilder = new DeciderGateBuilder(decider) @Provides @Singleton def providesConfig(configBuilder: ConfigBuilder): Config = configBuilder.build() } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/DiffyModule.scala ================================================ package com.twitter.follow_recommendations.modules import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.inject.annotations.Flag import com.twitter.decider.RandomRecipient import com.twitter.finagle.ThriftMux import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.thrift.ClientId import com.twitter.finatra.annotations.DarkTrafficService import com.twitter.follow_recommendations.configapi.deciders.DeciderKey import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService import com.twitter.inject.TwitterModule import com.twitter.inject.thrift.filters.DarkTrafficFilter import com.twitter.servo.decider.DeciderGateBuilder object DiffyModule extends TwitterModule { // diffy.dest is defined in the Follow Recommendations Service aurora file // and points to the Dark Traffic Proxy server private val destFlag = flag[String]("diffy.dest", "/$/nil", "Resolvable name of diffy-service or proxy") @Provides @Singleton @DarkTrafficService def provideDarkTrafficService( serviceIdentifier: ServiceIdentifier ): FollowRecommendationsThriftService.ReqRepServicePerEndpoint = { ThriftMux.client .withClientId(ClientId("follow_recos_service_darktraffic_proxy_client")) .withMutualTls(serviceIdentifier) .servicePerEndpoint[FollowRecommendationsThriftService.ReqRepServicePerEndpoint]( dest = destFlag(), label = "darktrafficproxy" ) } @Provides @Singleton def provideDarkTrafficFilter( @DarkTrafficService darkService: FollowRecommendationsThriftService.ReqRepServicePerEndpoint, deciderGateBuilder: DeciderGateBuilder, statsReceiver: StatsReceiver, @Flag("environment") env: String ): DarkTrafficFilter[FollowRecommendationsThriftService.ReqRepServicePerEndpoint] = { // sampleFunction is used to determine which requests should get replicated // to the dark traffic proxy server val sampleFunction: Any => Boolean = { _ => // check whether the current FRS instance is deployed in production env match { case "prod" => statsReceiver.scope("provideDarkTrafficFilter").counter("prod").incr() destFlag.isDefined && deciderGateBuilder .keyToFeature(DeciderKey.EnableTrafficDarkReading).isAvailable(RandomRecipient) case _ => statsReceiver.scope("provideDarkTrafficFilter").counter("devel").incr() // replicate zero requests if in non-production environment false } } new DarkTrafficFilter[FollowRecommendationsThriftService.ReqRepServicePerEndpoint]( darkService, sampleFunction, forwardAfterService = true, statsReceiver.scope("DarkTrafficFilter"), lookupByMethod = true ) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FeatureSwitchesModule.scala ================================================ package com.twitter.follow_recommendations.modules import com.google.inject.Provides import com.twitter.abdecider.LoggingABDecider import com.twitter.featureswitches.v2.Feature import com.twitter.featureswitches.v2.FeatureFilter import com.twitter.featureswitches.v2.FeatureSwitches import com.twitter.featureswitches.v2.builder.FeatureSwitchesBuilder import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants.PRODUCER_SIDE_FEATURE_SWITCHES import com.twitter.inject.TwitterModule import javax.inject.Named import javax.inject.Singleton object FeaturesSwitchesModule extends TwitterModule { private val DefaultConfigRepoPath = "/usr/local/config" private val FeaturesPath = "/features/onboarding/follow-recommendations-service/main" val isLocal = flag("configrepo.local", false, "Is the server running locally or in a DC") val localConfigRepoPath = flag( "local.configrepo", System.getProperty("user.home") + "/workspace/config", "Path to your local config repo" ) @Provides @Singleton def providesFeatureSwitches( abDecider: LoggingABDecider, statsReceiver: StatsReceiver ): FeatureSwitches = { val configRepoPath = if (isLocal()) { localConfigRepoPath() } else { DefaultConfigRepoPath } FeatureSwitchesBuilder .createDefault(FeaturesPath, abDecider, Some(statsReceiver)) .configRepoAbsPath(configRepoPath) .serviceDetailsFromAurora() .build() } @Provides @Singleton @Named(PRODUCER_SIDE_FEATURE_SWITCHES) def providesProducerFeatureSwitches( abDecider: LoggingABDecider, statsReceiver: StatsReceiver ): FeatureSwitches = { val configRepoPath = if (isLocal()) { localConfigRepoPath() } else { DefaultConfigRepoPath } /** * Feature Switches evaluate all tied FS Keys on Params construction time, which is very inefficient * for producer/candidate side holdbacks because we have 100s of candidates, and 100s of FS which result * in 10,000 FS evaluations when we want 1 per candidate (100 total), so we create a new FS Client * which has a [[ProducerFeatureFilter]] set for feature filter to reduce the FS Keys we evaluate. */ FeatureSwitchesBuilder .createDefault(FeaturesPath, abDecider, Some(statsReceiver.scope("producer_side_fs"))) .configRepoAbsPath(configRepoPath) .serviceDetailsFromAurora() .addFeatureFilter(ProducerFeatureFilter) .build() } } case object ProducerFeatureFilter extends FeatureFilter { private val AllowedKeys = Set( "post_nux_ml_flow_candidate_user_scorer_id", "frs_receiver_holdback_keep_social_user_candidate", "frs_receiver_holdback_keep_user_candidate") override def filter(feature: Feature): Option[Feature] = { if (AllowedKeys.exists(feature.parameters.contains)) { Some(feature) } else { None } } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FlagsModule.scala ================================================ package com.twitter.follow_recommendations.modules import com.twitter.inject.TwitterModule object FlagsModule extends TwitterModule { flag[Boolean]( name = "fetch_prod_promoted_accounts", help = "Whether or not to fetch production promoted accounts (true / false)" ) flag[Boolean]( name = "interests_opt_out_prod_enabled", help = "Whether to fetch intersts opt out data from the prod strato column or not" ) flag[Boolean]( name = "log_results", default = false, help = "Whether to log results such that we use them for scoring or metrics" ) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ProductRegistryModule.scala ================================================ package com.twitter.follow_recommendations.modules import com.twitter.follow_recommendations.products.ProdProductRegistry import com.twitter.follow_recommendations.products.common.ProductRegistry import com.twitter.inject.TwitterModule import javax.inject.Singleton object ProductRegistryModule extends TwitterModule { override protected def configure(): Unit = { bind[ProductRegistry].to[ProdProductRegistry].in[Singleton] } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScorerModule.scala ================================================ package com.twitter.follow_recommendations.modules import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.inject.TwitterModule import com.twitter.relevance.ep_model.common.CommonConstants import com.twitter.relevance.ep_model.scorer.EPScorer import com.twitter.relevance.ep_model.scorer.EPScorerBuilder import java.io.File import java.io.FileOutputStream import scala.language.postfixOps object ScorerModule extends TwitterModule { private val STPScorerPath = "/quality/stp_models/20141223" private def fileFromResource(resource: String): File = { val inputStream = getClass.getResourceAsStream(resource) val file = File.createTempFile(resource, "temp") val fos = new FileOutputStream(file) Iterator .continually(inputStream.read) .takeWhile(-1 !=) .foreach(fos.write) file } @Provides @Singleton def provideEpScorer: EPScorer = { val modelPath = fileFromResource(STPScorerPath + "/" + CommonConstants.EP_MODEL_FILE_NAME).getAbsolutePath val trainingConfigPath = fileFromResource(STPScorerPath + "/" + CommonConstants.TRAINING_CONFIG).getAbsolutePath val epScorer = new EPScorerBuilder epScorer .withModelPath(modelPath) .withTrainingConfig(trainingConfigPath) .build() } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScribeModule.scala ================================================ package com.twitter.follow_recommendations.modules import com.google.inject.Provides import com.google.inject.Singleton import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants import com.twitter.inject.TwitterModule import com.twitter.logging.BareFormatter import com.twitter.logging.HandlerFactory import com.twitter.logging.Level import com.twitter.logging.LoggerFactory import com.twitter.logging.NullHandler import com.twitter.logging.QueueingHandler import com.twitter.logging.ScribeHandler object ScribeModule extends TwitterModule { val useProdLogger = flag( name = "scribe.use_prod_loggers", default = false, help = "whether to use production logging for service" ) @Provides @Singleton @Named(GuiceNamedConstants.CLIENT_EVENT_LOGGER) def provideClientEventsLoggerFactory(stats: StatsReceiver): LoggerFactory = { val loggerCategory = "client_event" val clientEventsHandler: HandlerFactory = if (useProdLogger()) { QueueingHandler( maxQueueSize = 10000, handler = ScribeHandler( category = loggerCategory, formatter = BareFormatter, level = Some(Level.INFO), statsReceiver = stats.scope("client_event_scribe") ) ) } else { () => NullHandler } LoggerFactory( node = "abdecider", level = Some(Level.INFO), useParents = false, handlers = clientEventsHandler :: Nil ) } @Provides @Singleton @Named(GuiceNamedConstants.REQUEST_LOGGER) def provideFollowRecommendationsLoggerFactory(stats: StatsReceiver): LoggerFactory = { val loggerCategory = "follow_recommendations_logs" val handlerFactory: HandlerFactory = if (useProdLogger()) { QueueingHandler( maxQueueSize = 10000, handler = ScribeHandler( category = loggerCategory, formatter = BareFormatter, level = Some(Level.INFO), statsReceiver = stats.scope("follow_recommendations_logs_scribe") ) ) } else { () => NullHandler } LoggerFactory( node = loggerCategory, level = Some(Level.INFO), useParents = false, handlers = handlerFactory :: Nil ) } @Provides @Singleton @Named(GuiceNamedConstants.FLOW_LOGGER) def provideFrsRecommendationFlowLoggerFactory(stats: StatsReceiver): LoggerFactory = { val loggerCategory = "frs_recommendation_flow_logs" val handlerFactory: HandlerFactory = if (useProdLogger()) { QueueingHandler( maxQueueSize = 10000, handler = ScribeHandler( category = loggerCategory, formatter = BareFormatter, level = Some(Level.INFO), statsReceiver = stats.scope("frs_recommendation_flow_logs_scribe") ) ) } else { () => NullHandler } LoggerFactory( node = loggerCategory, level = Some(Level.INFO), useParents = false, handlers = handlerFactory :: Nil ) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/TimerModule.scala ================================================ package com.twitter.follow_recommendations.modules import com.google.inject.Provides import com.google.inject.Singleton import com.twitter.finagle.memcached.ZookeeperStateMonitor.DefaultTimer import com.twitter.inject.TwitterModule import com.twitter.util.Timer object TimerModule extends TwitterModule { @Provides @Singleton def providesTimer: Timer = DefaultTimer } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/slf4j:slf4j-api", "finatra/inject/inject-app/src/main/scala", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/ProdProductRegistry.scala ================================================ package com.twitter.follow_recommendations.products import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.follow_recommendations.products.common.ProductRegistry import com.twitter.follow_recommendations.products.explore_tab.ExploreTabProduct import com.twitter.follow_recommendations.products.home_timeline.HomeTimelineProduct import com.twitter.follow_recommendations.products.home_timeline_tweet_recs.HomeTimelineTweetRecsProduct import com.twitter.follow_recommendations.products.sidebar.SidebarProduct import javax.inject.Inject import javax.inject.Singleton @Singleton class ProdProductRegistry @Inject() ( exploreTabProduct: ExploreTabProduct, homeTimelineProduct: HomeTimelineProduct, homeTimelineTweetRecsProduct: HomeTimelineTweetRecsProduct, sidebarProduct: SidebarProduct, ) extends ProductRegistry { override val products: Seq[common.Product] = Seq( exploreTabProduct, homeTimelineProduct, homeTimelineTweetRecsProduct, sidebarProduct ) override val displayLocationProductMap: Map[DisplayLocation, common.Product] = products.groupBy(_.displayLocation).flatMap { case (loc, products) => assert(products.size == 1, s"Found more than 1 Product for ${loc}") products.headOption.map { product => loc -> product } } override def getProductByDisplayLocation(displayLocation: DisplayLocation): common.Product = { displayLocationProductMap.getOrElse( displayLocation, throw new MissingProductException(displayLocation)) } } class MissingProductException(displayLocation: DisplayLocation) extends Exception(s"No Product found for ${displayLocation}") ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "configapi/configapi-core", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Exceptions.scala ================================================ package com.twitter.follow_recommendations.products.common abstract class ProductException(message: String) extends Exception(message) class MissingFieldException(productRequest: ProductRequest, fieldName: String) extends ProductException( s"Missing ${fieldName} field for ${productRequest.recommendationRequest.displayLocation} request") ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Product.scala ================================================ package com.twitter.follow_recommendations.products.common import com.twitter.follow_recommendations.assembler.models.Layout import com.twitter.follow_recommendations.common.base.BaseRecommendationFlow import com.twitter.follow_recommendations.common.base.Transform import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.follow_recommendations.common.models.Recommendation import com.twitter.follow_recommendations.models.RecommendationRequest import com.twitter.product_mixer.core.model.marshalling.request.{Product => ProductMixerProduct} import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.Params trait Product { /** Each product also requires a human-readable name. * You can change this at any time */ def name: String /** * Every product needs a machine-friendly identifier for internal use. * You should use the same name as the product package name. * Except dashes are better than underscore * * Avoid changing this once it's in production. */ def identifier: String def displayLocation: DisplayLocation def selectWorkflows( request: ProductRequest ): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] /** * Blender is responsible for blending together the candidates generated by different flows used * in a product. For example, if a product uses two flows, it is blender's responsibility to * interleave their generated candidates together and make a unified sequence of candidates. */ def blender: Transform[ProductRequest, Recommendation] /** * It is resultsTransformer job to do any final transformations needed on the final list of * candidates generated by a product. For example, if a final quality check on candidates needed, * resultsTransformer will handle it. */ def resultsTransformer(request: ProductRequest): Stitch[Transform[ProductRequest, Recommendation]] def enabled(request: ProductRequest): Stitch[Boolean] def layout: Option[Layout] = None def productMixerProduct: Option[ProductMixerProduct] = None } case class ProductRequest(recommendationRequest: RecommendationRequest, params: Params) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/ProductRegistry.scala ================================================ package com.twitter.follow_recommendations.products.common import com.twitter.follow_recommendations.common.models.DisplayLocation trait ProductRegistry { def products: Seq[Product] def displayLocationProductMap: Map[DisplayLocation, Product] def getProductByDisplayLocation(displayLocation: DisplayLocation): Product } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "finatra/inject/inject-app/src/main/scala", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/ExploreTabProduct.scala ================================================ package com.twitter.follow_recommendations.products.explore_tab import com.twitter.follow_recommendations.common.base.BaseRecommendationFlow import com.twitter.follow_recommendations.common.base.IdentityTransform import com.twitter.follow_recommendations.common.base.Transform import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.follow_recommendations.common.models.Recommendation import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlFlow import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilder import com.twitter.follow_recommendations.products.common.Product import com.twitter.follow_recommendations.products.common.ProductRequest import com.twitter.follow_recommendations.products.explore_tab.configapi.ExploreTabParams import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class ExploreTabProduct @Inject() ( postNuxMlFlow: PostNuxMlFlow, postNuxMlRequestBuilder: PostNuxMlRequestBuilder) extends Product { override val name: String = "Explore Tab" override val identifier: String = "explore-tab" override val displayLocation: DisplayLocation = DisplayLocation.ExploreTab override def selectWorkflows( request: ProductRequest ): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] = { postNuxMlRequestBuilder.build(request).map { postNuxMlRequest => Seq(postNuxMlFlow.mapKey({ _: ProductRequest => postNuxMlRequest })) } } override val blender: Transform[ProductRequest, Recommendation] = new IdentityTransform[ProductRequest, Recommendation] override def resultsTransformer( request: ProductRequest ): Stitch[Transform[ProductRequest, Recommendation]] = Stitch.value(new IdentityTransform[ProductRequest, Recommendation]) override def enabled(request: ProductRequest): Stitch[Boolean] = { // Ideally we should hook up is_soft_user as custom FS field and disable the product through FS val enabledForUserType = !request.recommendationRequest.isSoftUser || request.params( ExploreTabParams.EnableProductForSoftUser) Stitch.value(request.params(ExploreTabParams.EnableProduct) && enabledForUserType) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "configapi/configapi-core", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabFSConfig.scala ================================================ package com.twitter.follow_recommendations.products.explore_tab.configapi import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.follow_recommendations.products.explore_tab.configapi.ExploreTabParams._ import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.Param import javax.inject.Inject import javax.inject.Singleton @Singleton class ExploreTabFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq(EnableProductForSoftUser) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabParams.scala ================================================ package com.twitter.follow_recommendations.products.explore_tab.configapi import com.twitter.timelines.configapi.Param import com.twitter.timelines.configapi.FSParam object ExploreTabParams { object EnableProduct extends Param[Boolean](false) object EnableProductForSoftUser extends FSParam[Boolean]("explore_tab_enable_product_for_soft_user", false) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "finatra/inject/inject-app/src/main/scala", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HTLProductMixer.scala ================================================ package com.twitter.follow_recommendations.products.home_timeline import com.twitter.product_mixer.core.model.common.identifier.ProductIdentifier import com.twitter.product_mixer.core.model.marshalling.request.Product case object HTLProductMixer extends Product { override val identifier: ProductIdentifier = ProductIdentifier("HomeTimeline") override val stringCenterProject: Option[String] = Some("people-discovery") } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineProduct.scala ================================================ package com.twitter.follow_recommendations.products.home_timeline import com.twitter.follow_recommendations.assembler.models.ActionConfig import com.twitter.follow_recommendations.assembler.models.FollowedByUsersProof import com.twitter.follow_recommendations.assembler.models.FooterConfig import com.twitter.follow_recommendations.assembler.models.GeoContextProof import com.twitter.follow_recommendations.assembler.models.HeaderConfig import com.twitter.follow_recommendations.assembler.models.Layout import com.twitter.follow_recommendations.assembler.models.TitleConfig import com.twitter.follow_recommendations.assembler.models.UserListLayout import com.twitter.follow_recommendations.assembler.models.UserListOptions import com.twitter.follow_recommendations.common.base.BaseRecommendationFlow import com.twitter.follow_recommendations.common.base.IdentityTransform import com.twitter.follow_recommendations.common.base.Transform import com.twitter.follow_recommendations.flows.ads.PromotedAccountsFlow import com.twitter.follow_recommendations.flows.ads.PromotedAccountsFlowRequest import com.twitter.follow_recommendations.blenders.PromotedAccountsBlender import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.follow_recommendations.common.models.Recommendation import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlFlow import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilder import com.twitter.follow_recommendations.products.common.Product import com.twitter.follow_recommendations.products.common.ProductRequest import com.twitter.follow_recommendations.products.home_timeline.configapi.HomeTimelineParams._ import com.twitter.inject.Injector import com.twitter.product_mixer.core.model.marshalling.request import com.twitter.product_mixer.core.product.guice.ProductScope import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class HomeTimelineProduct @Inject() ( postNuxMlFlow: PostNuxMlFlow, postNuxMlRequestBuilder: PostNuxMlRequestBuilder, promotedAccountsFlow: PromotedAccountsFlow, promotedAccountsBlender: PromotedAccountsBlender, productScope: ProductScope, injector: Injector, ) extends Product { override val name: String = "Home Timeline" override val identifier: String = "home-timeline" override val displayLocation: DisplayLocation = DisplayLocation.HomeTimeline override def selectWorkflows( request: ProductRequest ): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] = { postNuxMlRequestBuilder.build(request).map { postNuxMlRequest => Seq( postNuxMlFlow.mapKey({ request: ProductRequest => postNuxMlRequest }), promotedAccountsFlow.mapKey(mkPromotedAccountsRequest)) } } override val blender: Transform[ProductRequest, Recommendation] = { promotedAccountsBlender.mapTarget[ProductRequest](getMaxResults) } private val identityTransform = new IdentityTransform[ProductRequest, Recommendation] override def resultsTransformer( request: ProductRequest ): Stitch[Transform[ProductRequest, Recommendation]] = Stitch.value(identityTransform) override def enabled(request: ProductRequest): Stitch[Boolean] = Stitch.value(request.params(EnableProduct)) override def layout: Option[Layout] = { productMixerProduct.map { product => val homeTimelineStrings = productScope.let(product) { injector.instance[HomeTimelineStrings] } UserListLayout( header = Some(HeaderConfig(TitleConfig(homeTimelineStrings.whoToFollowModuleTitle))), userListOptions = UserListOptions(userBioEnabled = true, userBioTruncated = true, None), socialProofs = Some( Seq( FollowedByUsersProof( homeTimelineStrings.whoToFollowFollowedByManyUserSingleString, homeTimelineStrings.whoToFollowFollowedByManyUserDoubleString, homeTimelineStrings.whoToFollowFollowedByManyUserMultipleString ), GeoContextProof(homeTimelineStrings.whoToFollowPopularInCountryKey) )), footer = Some( FooterConfig( Some(ActionConfig(homeTimelineStrings.whoToFollowModuleFooter, "http://twitter.com")))) ) } } override def productMixerProduct: Option[request.Product] = Some(HTLProductMixer) private[home_timeline] def mkPromotedAccountsRequest( req: ProductRequest ): PromotedAccountsFlowRequest = { PromotedAccountsFlowRequest( req.recommendationRequest.clientContext, req.params, req.recommendationRequest.displayLocation, None, req.recommendationRequest.excludedIds.getOrElse(Nil) ) } private[home_timeline] def getMaxResults(req: ProductRequest): Int = { req.recommendationRequest.maxResults.getOrElse( req.params(DefaultMaxResults) ) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineStrings.scala ================================================ package com.twitter.follow_recommendations.products.home_timeline import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.ExternalStringRegistry import com.twitter.stringcenter.client.core.ExternalString import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton class HomeTimelineStrings @Inject() ( @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) { private val externalStringRegistry = externalStringRegistryProvider.get() val whoToFollowFollowedByManyUserSingleString: ExternalString = externalStringRegistry.createProdString("WtfRecommendationContext.followedByManyUserSingle") val whoToFollowFollowedByManyUserDoubleString: ExternalString = externalStringRegistry.createProdString("WtfRecommendationContext.followedByManyUserDouble") val whoToFollowFollowedByManyUserMultipleString: ExternalString = externalStringRegistry.createProdString("WtfRecommendationContext.followedByManyUserMultiple") val whoToFollowPopularInCountryKey: ExternalString = externalStringRegistry.createProdString("WtfRecommendationContext.popularInCountry") val whoToFollowModuleTitle: ExternalString = externalStringRegistry.createProdString("WhoToFollowModule.title") val whoToFollowModuleFooter: ExternalString = externalStringRegistry.createProdString("WhoToFollowModule.pivot") } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "configapi/configapi-core", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineFSConfig.scala ================================================ package com.twitter.follow_recommendations.products.home_timeline.configapi import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig import com.twitter.follow_recommendations.products.home_timeline.configapi.HomeTimelineParams._ import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.timelines.configapi.Param import com.twitter.util.Duration import javax.inject.Inject import javax.inject.Singleton @Singleton class HomeTimelineFSConfig @Inject() () extends FeatureSwitchConfig { override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq(EnableWritingServingHistory) override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( DurationGuardrailToForceSuggest, SuggestBasedFatigueDuration ) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineParams.scala ================================================ package com.twitter.follow_recommendations.products.home_timeline.configapi import com.twitter.conversions.DurationOps._ import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.timelines.configapi.Param import com.twitter.util.Duration object HomeTimelineParams { object EnableProduct extends Param[Boolean](false) object DefaultMaxResults extends Param[Int](20) object EnableWritingServingHistory extends FSParam[Boolean]("home_timeline_enable_writing_serving_history", false) object DurationGuardrailToForceSuggest extends FSBoundedParam[Duration]( name = "home_timeline_duration_guardrail_to_force_suggest_in_hours", default = 0.hours, min = 0.hours, max = 1000.hours) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromHours } object SuggestBasedFatigueDuration extends FSBoundedParam[Duration]( name = "home_timeline_suggest_based_fatigue_duration_in_hours", default = 0.hours, min = 0.hours, max = 1000.hours) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromHours } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "finatra/inject/inject-app/src/main/scala", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/HomeTimelineTweetRecsProduct.scala ================================================ package com.twitter.follow_recommendations.products.home_timeline_tweet_recs import com.twitter.follow_recommendations.common.base.BaseRecommendationFlow import com.twitter.follow_recommendations.common.base.IdentityTransform import com.twitter.follow_recommendations.common.base.Transform import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.follow_recommendations.common.models.Recommendation import com.twitter.follow_recommendations.flows.content_recommender_flow.ContentRecommenderFlow import com.twitter.follow_recommendations.flows.content_recommender_flow.ContentRecommenderRequestBuilder import com.twitter.follow_recommendations.products.common.Product import com.twitter.follow_recommendations.products.common.ProductRequest import com.twitter.follow_recommendations.products.home_timeline_tweet_recs.configapi.HomeTimelineTweetRecsParams._ import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton /* * This "DisplayLocation" is used to generate user recommendations using the ContentRecommenderFlow. These recommendations are later used downstream * to generate recommended tweets on Home Timeline. */ @Singleton class HomeTimelineTweetRecsProduct @Inject() ( contentRecommenderFlow: ContentRecommenderFlow, contentRecommenderRequestBuilder: ContentRecommenderRequestBuilder) extends Product { override val name: String = "Home Timeline Tweet Recs" override val identifier: String = "home-timeline-tweet-recs" override val displayLocation: DisplayLocation = DisplayLocation.HomeTimelineTweetRecs override def selectWorkflows( request: ProductRequest ): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] = { contentRecommenderRequestBuilder.build(request).map { contentRecommenderRequest => Seq(contentRecommenderFlow.mapKey({ request: ProductRequest => contentRecommenderRequest })) } } override val blender: Transform[ProductRequest, Recommendation] = new IdentityTransform[ProductRequest, Recommendation] override def resultsTransformer( request: ProductRequest ): Stitch[Transform[ProductRequest, Recommendation]] = Stitch.value(new IdentityTransform[ProductRequest, Recommendation]) override def enabled(request: ProductRequest): Stitch[Boolean] = Stitch.value(request.params(EnableProduct)) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "configapi/configapi-core", "configapi/configapi-decider", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/HomeTimelineTweetRecsParams.scala ================================================ package com.twitter.follow_recommendations.products.home_timeline_tweet_recs.configapi import com.twitter.timelines.configapi.Param object HomeTimelineTweetRecsParams { object EnableProduct extends Param[Boolean](false) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "finatra/inject/inject-app/src/main/scala", "finatra/inject/inject-core/src/main/scala", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/SidebarProduct.scala ================================================ package com.twitter.follow_recommendations.products.sidebar import com.twitter.follow_recommendations.common.base.BaseRecommendationFlow import com.twitter.follow_recommendations.common.base.IdentityTransform import com.twitter.follow_recommendations.common.base.Transform import com.twitter.follow_recommendations.flows.ads.PromotedAccountsFlow import com.twitter.follow_recommendations.flows.ads.PromotedAccountsFlowRequest import com.twitter.follow_recommendations.blenders.PromotedAccountsBlender import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.follow_recommendations.common.models.Recommendation import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlFlow import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilder import com.twitter.follow_recommendations.products.common.Product import com.twitter.follow_recommendations.products.common.ProductRequest import com.twitter.follow_recommendations.products.sidebar.configapi.SidebarParams import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class SidebarProduct @Inject() ( postNuxMlFlow: PostNuxMlFlow, postNuxMlRequestBuilder: PostNuxMlRequestBuilder, promotedAccountsFlow: PromotedAccountsFlow, promotedAccountsBlender: PromotedAccountsBlender) extends Product { override val name: String = "Sidebar" override val identifier: String = "sidebar" override val displayLocation: DisplayLocation = DisplayLocation.Sidebar override def selectWorkflows( request: ProductRequest ): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] = { postNuxMlRequestBuilder.build(request).map { postNuxMlRequest => Seq( postNuxMlFlow.mapKey({ _: ProductRequest => postNuxMlRequest }), promotedAccountsFlow.mapKey(mkPromotedAccountsRequest) ) } } override val blender: Transform[ProductRequest, Recommendation] = { promotedAccountsBlender.mapTarget[ProductRequest](getMaxResults) } private[sidebar] def mkPromotedAccountsRequest( req: ProductRequest ): PromotedAccountsFlowRequest = { PromotedAccountsFlowRequest( req.recommendationRequest.clientContext, req.params, req.recommendationRequest.displayLocation, None, req.recommendationRequest.excludedIds.getOrElse(Nil) ) } private[sidebar] def getMaxResults(req: ProductRequest): Int = { req.recommendationRequest.maxResults.getOrElse( req.params(SidebarParams.DefaultMaxResults) ) } override def resultsTransformer( request: ProductRequest ): Stitch[Transform[ProductRequest, Recommendation]] = Stitch.value(new IdentityTransform[ProductRequest, Recommendation]) override def enabled(request: ProductRequest): Stitch[Boolean] = Stitch.value(request.params(SidebarParams.EnableProduct)) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "configapi/configapi-core", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/SidebarParams.scala ================================================ package com.twitter.follow_recommendations.products.sidebar.configapi import com.twitter.timelines.configapi.Param object SidebarParams { object EnableProduct extends Param[Boolean](false) object DefaultMaxResults extends Param[Int](20) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], tags = ["bazel-compatible"], dependencies = [ "finatra/inject/inject-app/src/main/scala", "finatra/inject/inject-core/src/main/scala", "finatra/inject/inject-server/src/main/scala", "finatra/inject/inject-thrift-client", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products", "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", "twitter-server/server/src/main/scala", "util/util-app/src/main/scala", "util/util-core:scala", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/FollowRecommendationsServiceWarmupHandler.scala ================================================ package com.twitter.follow_recommendations.services import com.twitter.finagle.thrift.ClientId import com.twitter.finatra.thrift.routing.ThriftWarmup import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService.GetRecommendations import com.twitter.follow_recommendations.thriftscala.ClientContext import com.twitter.follow_recommendations.thriftscala.DebugParams import com.twitter.follow_recommendations.thriftscala.DisplayContext import com.twitter.follow_recommendations.thriftscala.DisplayLocation import com.twitter.follow_recommendations.thriftscala.Profile import com.twitter.follow_recommendations.thriftscala.RecommendationRequest import com.twitter.inject.Logging import com.twitter.inject.utils.Handler import com.twitter.scrooge.Request import com.twitter.scrooge.Response import com.twitter.util.Return import com.twitter.util.Throw import com.twitter.util.Try import javax.inject.Inject import javax.inject.Singleton @Singleton class FollowRecommendationsServiceWarmupHandler @Inject() (warmup: ThriftWarmup) extends Handler with Logging { private val clientId = ClientId("thrift-warmup-client") override def handle(): Unit = { val testIds = Seq(1L) def warmupQuery(userId: Long, displayLocation: DisplayLocation): RecommendationRequest = { val clientContext = ClientContext( userId = Some(userId), guestId = None, appId = Some(258901L), ipAddress = Some("0.0.0.0"), userAgent = Some("FAKE_USER_AGENT_FOR_WARMUPS"), countryCode = Some("US"), languageCode = Some("en"), isTwoffice = None, userRoles = None, deviceId = Some("FAKE_DEVICE_ID_FOR_WARMUPS") ) RecommendationRequest( clientContext = clientContext, displayLocation = displayLocation, displayContext = None, maxResults = Some(3), fetchPromotedContent = Some(false), debugParams = Some(DebugParams(doNotLog = Some(true))) ) } // Add FRS display locations here if they should be targeted for warm-up // when FRS is starting from a fresh state after a deploy val displayLocationsToWarmUp: Seq[DisplayLocation] = Seq( DisplayLocation.HomeTimeline, DisplayLocation.HomeTimelineReverseChron, DisplayLocation.ProfileSidebar, DisplayLocation.NuxInterests, DisplayLocation.NuxPymk ) try { clientId.asCurrent { // Iterate over each user ID created for testing testIds foreach { id => // Iterate over each display location targeted for warm-up displayLocationsToWarmUp foreach { displayLocation => val warmupReq = warmupQuery(id, displayLocation) info(s"Sending warm-up request to service with query: $warmupReq") warmup.sendRequest( method = GetRecommendations, req = Request(GetRecommendations.Args(warmupReq)))(assertWarmupResponse) // send the request one more time so that it goes through cache hits warmup.sendRequest( method = GetRecommendations, req = Request(GetRecommendations.Args(warmupReq)))(assertWarmupResponse) } } } } catch { case e: Throwable => // we don't want a warmup failure to prevent start-up error(e.getMessage, e) } info("Warm-up done.") } /* Private */ private def assertWarmupResponse(result: Try[Response[GetRecommendations.SuccessType]]): Unit = { // we collect and log any exceptions from the result. result match { case Return(_) => // ok case Throw(exception) => warn() error(s"Error performing warm-up request: ${exception.getMessage}", exception) } } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductMixerRecommendationService.scala ================================================ package com.twitter.follow_recommendations.services import com.twitter.finagle.stats.StatsReceiver import javax.inject.Inject import javax.inject.Singleton import com.twitter.timelines.configapi.Params import com.twitter.follow_recommendations.common.utils.DisplayLocationProductConverterUtil import com.twitter.follow_recommendations.configapi.deciders.DeciderParams import com.twitter.follow_recommendations.logging.FrsLogger import com.twitter.follow_recommendations.models.{DebugParams => FrsDebugParams} import com.twitter.follow_recommendations.models.RecommendationRequest import com.twitter.follow_recommendations.models.RecommendationResponse import com.twitter.follow_recommendations.models.Request import com.twitter.product_mixer.core.model.marshalling.request.{ DebugParams => ProductMixerDebugParams } import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest import com.twitter.stitch.Stitch @Singleton class ProductMixerRecommendationService @Inject() ( productPipelineRegistry: ProductPipelineRegistry, resultLogger: FrsLogger, baseStats: StatsReceiver) { private val stats = baseStats.scope("product_mixer_recos_service_stats") private val loggingStats = stats.scope("logged") def get(request: RecommendationRequest, params: Params): Stitch[RecommendationResponse] = { if (params(DeciderParams.EnableRecommendations)) { val productMixerRequest = convertToProductMixerRequest(request) productPipelineRegistry .getProductPipeline[Request, RecommendationResponse](productMixerRequest.product) .process(ProductPipelineRequest(productMixerRequest, params)).onSuccess { response => if (resultLogger.shouldLog(request.debugParams)) { loggingStats.counter().incr() resultLogger.logRecommendationResult(request, response) } } } else { Stitch.value(RecommendationResponse(Nil)) } } def convertToProductMixerRequest(frsRequest: RecommendationRequest): Request = { Request( maxResults = frsRequest.maxResults, debugParams = convertToProductMixerDebugParams(frsRequest.debugParams), productContext = None, product = DisplayLocationProductConverterUtil.displayLocationToProduct(frsRequest.displayLocation), clientContext = frsRequest.clientContext, serializedRequestCursor = frsRequest.cursor, frsDebugParams = frsRequest.debugParams, displayLocation = frsRequest.displayLocation, excludedIds = frsRequest.excludedIds, fetchPromotedContent = frsRequest.fetchPromotedContent, userLocationState = frsRequest.userLocationState ) } private def convertToProductMixerDebugParams( frsDebugParams: Option[FrsDebugParams] ): Option[ProductMixerDebugParams] = { frsDebugParams.map { debugParams => ProductMixerDebugParams(debugParams.featureOverrides, None) } } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelector.scala ================================================ package com.twitter.follow_recommendations.services import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.StatsUtil import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.common.models.DebugOptions import com.twitter.follow_recommendations.models.DebugParams import com.twitter.follow_recommendations.models.RecommendationRequest import com.twitter.follow_recommendations.models.RecommendationResponse import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.Params import javax.inject.Inject import javax.inject.Singleton import scala.util.Random @Singleton class ProductPipelineSelector @Inject() ( recommendationsService: RecommendationsService, productMixerRecommendationService: ProductMixerRecommendationService, productPipelineSelectorConfig: ProductPipelineSelectorConfig, baseStats: StatsReceiver) { private val frsStats = baseStats.scope("follow_recommendations_service") private val stats = frsStats.scope("product_pipeline_selector_parity") private val readFromProductMixerCounter = stats.counter("select_product_mixer") private val readFromOldFRSCounter = stats.counter("select_old_frs") def selectPipeline( request: RecommendationRequest, params: Params ): Stitch[RecommendationResponse] = { productPipelineSelectorConfig .getDarkReadAndExpParams(request.displayLocation).map { darkReadAndExpParam => if (params(darkReadAndExpParam.expParam)) { readFromProductMixerPipeline(request, params) } else if (params(darkReadAndExpParam.darkReadParam)) { darkReadAndReturnResult(request, params) } else { readFromOldFrsPipeline(request, params) } }.getOrElse(readFromOldFrsPipeline(request, params)) } private def readFromProductMixerPipeline( request: RecommendationRequest, params: Params ): Stitch[RecommendationResponse] = { readFromProductMixerCounter.incr() productMixerRecommendationService.get(request, params) } private def readFromOldFrsPipeline( request: RecommendationRequest, params: Params ): Stitch[RecommendationResponse] = { readFromOldFRSCounter.incr() recommendationsService.get(request, params) } private def darkReadAndReturnResult( request: RecommendationRequest, params: Params ): Stitch[RecommendationResponse] = { val darkReadStats = stats.scope("select_dark_read", request.displayLocation.toFsName) darkReadStats.counter("count").incr() // If no seed is set, create a random one that both requests will use to remove differences // in randomness for the WeightedCandidateSourceRanker val randomizationSeed = new Random().nextLong() val oldFRSPiplelineRequest = request.copy( debugParams = Some( request.debugParams.getOrElse( DebugParams(None, Some(DebugOptions(randomizationSeed = Some(randomizationSeed)))))) ) val productMixerPipelineRequest = request.copy( debugParams = Some( request.debugParams.getOrElse( DebugParams( None, Some(DebugOptions(doNotLog = true, randomizationSeed = Some(randomizationSeed)))))) ) StatsUtil .profileStitch( readFromOldFrsPipeline(oldFRSPiplelineRequest, params), darkReadStats.scope("frs_timing")).applyEffect { frsOldPipelineResponse => Stitch.async( StatsUtil .profileStitch( readFromProductMixerPipeline(productMixerPipelineRequest, params), darkReadStats.scope("product_mixer_timing")).liftToOption().map { case Some(frsProductMixerResponse) => darkReadStats.counter("product_mixer_pipeline_success").incr() compare(request, frsOldPipelineResponse, frsProductMixerResponse) case None => darkReadStats.counter("product_mixer_pipeline_failure").incr() } ) } } def compare( request: RecommendationRequest, frsOldPipelineResponse: RecommendationResponse, frsProductMixerResponse: RecommendationResponse ): Unit = { val compareStats = stats.scope("pipeline_comparison", request.displayLocation.toFsName) compareStats.counter("total-comparisons").incr() val oldFrsMap = frsOldPipelineResponse.recommendations.map { user => user.id -> user }.toMap val productMixerMap = frsProductMixerResponse.recommendations.map { user => user.id -> user }.toMap compareTopNResults(3, frsOldPipelineResponse, frsProductMixerResponse, compareStats) compareTopNResults(5, frsOldPipelineResponse, frsProductMixerResponse, compareStats) compareTopNResults(25, frsOldPipelineResponse, frsProductMixerResponse, compareStats) compareTopNResults(50, frsOldPipelineResponse, frsProductMixerResponse, compareStats) compareTopNResults(75, frsOldPipelineResponse, frsProductMixerResponse, compareStats) // Compare individual matching candidates oldFrsMap.keys.foreach(userId => { if (productMixerMap.contains(userId)) { (oldFrsMap(userId), productMixerMap(userId)) match { case (oldFrsUser: CandidateUser, productMixerUser: CandidateUser) => compareStats.counter("matching-user-count").incr() compareUser(oldFrsUser, productMixerUser, compareStats) case _ => compareStats.counter("unknown-user-type-count").incr() } } else { compareStats.counter("missing-user-count").incr() } }) } private def compareTopNResults( n: Int, frsOldPipelineResponse: RecommendationResponse, frsProductMixerResponse: RecommendationResponse, compareStats: StatsReceiver ): Unit = { if (frsOldPipelineResponse.recommendations.size >= n && frsProductMixerResponse.recommendations.size >= n) { val oldFrsPipelineFirstN = frsOldPipelineResponse.recommendations.take(n).map(_.id) val productMixerPipelineFirstN = frsProductMixerResponse.recommendations.take(n).map(_.id) if (oldFrsPipelineFirstN.sorted == productMixerPipelineFirstN.sorted) compareStats.counter(s"first-$n-sorted-equal-ids").incr() if (oldFrsPipelineFirstN == productMixerPipelineFirstN) compareStats.counter(s"first-$n-unsorted-ids-equal").incr() else compareStats.counter(s"first-$n-unsorted-ids-unequal").incr() } } private def compareUser( oldFrsUser: CandidateUser, productMixerUser: CandidateUser, stats: StatsReceiver ): Unit = { val userStats = stats.scope("matching-user") if (oldFrsUser.score != productMixerUser.score) userStats.counter("mismatch-score").incr() if (oldFrsUser.reason != productMixerUser.reason) userStats.counter("mismatch-reason").incr() if (oldFrsUser.userCandidateSourceDetails != productMixerUser.userCandidateSourceDetails) userStats.counter("mismatch-userCandidateSourceDetails").incr() if (oldFrsUser.adMetadata != productMixerUser.adMetadata) userStats.counter("mismatch-adMetadata").incr() if (oldFrsUser.trackingToken != productMixerUser.trackingToken) userStats.counter("mismatch-trackingToken").incr() if (oldFrsUser.dataRecord != productMixerUser.dataRecord) userStats.counter("mismatch-dataRecord").incr() if (oldFrsUser.scores != productMixerUser.scores) userStats.counter("mismatch-scores").incr() if (oldFrsUser.infoPerRankingStage != productMixerUser.infoPerRankingStage) userStats.counter("mismatch-infoPerRankingStage").incr() if (oldFrsUser.params != productMixerUser.params) userStats.counter("mismatch-params").incr() if (oldFrsUser.engagements != productMixerUser.engagements) userStats.counter("mismatch-engagements").incr() if (oldFrsUser.recommendationFlowIdentifier != productMixerUser.recommendationFlowIdentifier) userStats.counter("mismatch-recommendationFlowIdentifier").incr() } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelectorConfig.scala ================================================ package com.twitter.follow_recommendations.services import com.twitter.follow_recommendations.common.models.DisplayLocation import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.Param import javax.inject.Singleton @Singleton class ProductPipelineSelectorConfig { private val paramsMap: Map[DisplayLocation, DarkReadAndExpParams] = Map.empty def getDarkReadAndExpParams( displayLocation: DisplayLocation ): Option[DarkReadAndExpParams] = { paramsMap.get(displayLocation) } } case class DarkReadAndExpParams(darkReadParam: Param[Boolean], expParam: FSParam[Boolean]) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductRecommenderService.scala ================================================ package com.twitter.follow_recommendations.services import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.StatsUtil import com.twitter.follow_recommendations.common.models.Recommendation import com.twitter.follow_recommendations.models.RecommendationRequest import com.twitter.follow_recommendations.products.common.ProductRegistry import com.twitter.follow_recommendations.products.common.ProductRequest import com.twitter.stitch.Stitch import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableWhoToFollowProducts import com.twitter.timelines.configapi.Params import javax.inject.Inject import javax.inject.Singleton @Singleton class ProductRecommenderService @Inject() ( productRegistry: ProductRegistry, statsReceiver: StatsReceiver) { private val stats = statsReceiver.scope("ProductRecommenderService") def getRecommendations( request: RecommendationRequest, params: Params ): Stitch[Seq[Recommendation]] = { val displayLocation = request.displayLocation val displayLocationStatName = displayLocation.toString val locationStats = stats.scope(displayLocationStatName) val loggedInOrOutStats = if (request.clientContext.userId.isDefined) { stats.scope("logged_in").scope(displayLocationStatName) } else { stats.scope("logged_out").scope(displayLocationStatName) } loggedInOrOutStats.counter("requests").incr() val product = productRegistry.getProductByDisplayLocation(displayLocation) val productRequest = ProductRequest(request, params) val productEnabledStitch = StatsUtil.profileStitch(product.enabled(productRequest), locationStats.scope("enabled")) productEnabledStitch.flatMap { productEnabled => if (productEnabled && params(EnableWhoToFollowProducts)) { loggedInOrOutStats.counter("enabled").incr() val stitch = for { workflows <- StatsUtil.profileStitch( product.selectWorkflows(productRequest), locationStats.scope("select_workflows")) workflowRecos <- StatsUtil.profileStitch( Stitch.collect( workflows.map(_.process(productRequest).map(_.result.getOrElse(Seq.empty)))), locationStats.scope("execute_workflows") ) blendedCandidates <- StatsUtil.profileStitch( product.blender.transform(productRequest, workflowRecos.flatten), locationStats.scope("blend_results")) resultsTransformer <- StatsUtil.profileStitch( product.resultsTransformer(productRequest), locationStats.scope("results_transformer")) transformedCandidates <- StatsUtil.profileStitch( resultsTransformer.transform(productRequest, blendedCandidates), locationStats.scope("execute_results_transformer")) } yield { transformedCandidates } StatsUtil.profileStitchResults[Seq[Recommendation]](stitch, locationStats, _.size) } else { loggedInOrOutStats.counter("disabled").incr() locationStats.counter("disabled_product").incr() Stitch.Nil } } } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/RecommendationsService.scala ================================================ package com.twitter.follow_recommendations.services import com.twitter.follow_recommendations.configapi.deciders.DeciderParams import com.twitter.follow_recommendations.logging.FrsLogger import com.twitter.follow_recommendations.models.RecommendationRequest import com.twitter.follow_recommendations.models.RecommendationResponse import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.Params import javax.inject.Inject import javax.inject.Singleton @Singleton class RecommendationsService @Inject() ( productRecommenderService: ProductRecommenderService, resultLogger: FrsLogger) { def get(request: RecommendationRequest, params: Params): Stitch[RecommendationResponse] = { if (params(DeciderParams.EnableRecommendations)) { productRecommenderService .getRecommendations(request, params).map(RecommendationResponse).onSuccess { response => if (resultLogger.shouldLog(request.debugParams)) { resultLogger.logRecommendationResult(request, response) } } } else { Stitch.value(RecommendationResponse(Nil)) } } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/UserScoringService.scala ================================================ package com.twitter.follow_recommendations.services import com.twitter.finagle.stats.Counter import com.twitter.finagle.stats.StatsReceiver import com.twitter.follow_recommendations.common.base.StatsUtil.profileStitchSeqResults import com.twitter.follow_recommendations.common.clients.impression_store.WtfImpressionStore import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.HydrateFeaturesTransform import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.MlRanker import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStats import com.twitter.follow_recommendations.configapi.deciders.DeciderParams import com.twitter.follow_recommendations.logging.FrsLogger import com.twitter.follow_recommendations.models.ScoringUserRequest import com.twitter.follow_recommendations.models.ScoringUserResponse import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class UserScoringService @Inject() ( socialGraph: SocialGraphClient, wtfImpressionStore: WtfImpressionStore, hydrateFeaturesTransform: HydrateFeaturesTransform[ScoringUserRequest], mlRanker: MlRanker[ScoringUserRequest], resultLogger: FrsLogger, stats: StatsReceiver) { private val scopedStats: StatsReceiver = stats.scope(this.getClass.getSimpleName) private val disabledCounter: Counter = scopedStats.counter("disabled") def get(request: ScoringUserRequest): Stitch[ScoringUserResponse] = { if (request.params(DeciderParams.EnableScoreUserCandidates)) { val hydratedRequest = hydrate(request) val candidatesStitch = hydratedRequest.flatMap { req => hydrateFeaturesTransform.transform(req, request.candidates).flatMap { candidateWithFeatures => mlRanker.rank(req, candidateWithFeatures) } } profileStitchSeqResults(candidatesStitch, scopedStats) .map(ScoringUserResponse) .onSuccess { response => if (resultLogger.shouldLog(request.debugParams)) { resultLogger.logScoringResult(request, response) } } } else { disabledCounter.incr() Stitch.value(ScoringUserResponse(Nil)) } } private def hydrate(request: ScoringUserRequest): Stitch[ScoringUserRequest] = { val allStitches = Stitch.collect(request.clientContext.userId.map { userId => val recentFollowedUserIdsStitch = rescueWithStats( socialGraph.getRecentFollowedUserIds(userId), stats, "recentFollowedUserIds") val recentFollowedByUserIdsStitch = rescueWithStats( socialGraph.getRecentFollowedByUserIds(userId), stats, "recentFollowedByUserIds") val wtfImpressionsStitch = rescueWithStats( wtfImpressionStore.get(userId, request.displayLocation), stats, "wtfImpressions") Stitch.join(recentFollowedUserIdsStitch, recentFollowedByUserIdsStitch, wtfImpressionsStitch) }) allStitches.map { case Some((recentFollowedUserIds, recentFollowedByUserIds, wtfImpressions)) => request.copy( recentFollowedUserIds = if (recentFollowedUserIds.isEmpty) None else Some(recentFollowedUserIds), recentFollowedByUserIds = if (recentFollowedByUserIds.isEmpty) None else Some(recentFollowedByUserIds), wtfImpressions = if (wtfImpressions.isEmpty) None else Some(wtfImpressions) ) case _ => request } } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/BUILD ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], tags = ["bazel-compatible"], dependencies = [ "finatra/thrift/src/main/scala/com/twitter/finatra/thrift", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift:controller", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/exceptions", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/filters", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/modules", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/response", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/routing", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/UnknownExceptionMapper.scala ================================================ package com.twitter.follow_recommendations.service.exceptions import com.twitter.finatra.thrift.exceptions.ExceptionMapper import com.twitter.inject.Logging import com.twitter.util.Future import javax.inject.Singleton @Singleton class UnknownLoggingExceptionMapper extends ExceptionMapper[Exception, Throwable] with Logging { def handleException(throwable: Exception): Future[Throwable] = { error( s"Unmapped Exception: ${throwable.getMessage} - ${throwable.getStackTrace.mkString(", \n\t")}", throwable ) Future.exception(throwable) } } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/BUILD ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "finatra/inject/inject-app/src/main/scala", "finatra/inject/inject-core/src/main/scala", "finatra/inject/inject-server/src/main/scala", "finatra/inject/inject-thrift-client", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk", "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", "twitter-server/server/src/main/scala", "util/util-app/src/main/scala", "util/util-core:scala", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/CandidateSourceHoldbackUtil.scala ================================================ package com.twitter.follow_recommendations.utils import com.twitter.follow_recommendations.common.candidate_sources.addressbook._ import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeoSource import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RecentEngagementNonDirectFollowSource import com.twitter.follow_recommendations.common.candidate_sources.sims.SwitchingSimsSource import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentStrongEngagementDirectFollowSimilarUsersSource import com.twitter.follow_recommendations.common.candidate_sources.socialgraph.RecentFollowingRecentFollowingExpansionSource import com.twitter.follow_recommendations.common.candidate_sources.stp.MutualFollowStrongTiePredictionSource import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource import com.twitter.follow_recommendations.common.candidate_sources.stp.BaseOnlineSTPSource import com.twitter.follow_recommendations.common.candidate_sources.stp.SocialProofEnforcedOfflineStrongTiePredictionSource import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource import com.twitter.follow_recommendations.common.candidate_sources.two_hop_random_walk.TwoHopRandomWalkSource import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.follow_recommendations.configapi.params.GlobalParams import com.twitter.follow_recommendations.models.CandidateSourceType import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.timelines.configapi.HasParams trait CandidateSourceHoldbackUtil { import CandidateSourceHoldbackUtil._ def filterCandidateSources[T <: HasParams]( request: T, sources: Seq[CandidateSource[T, CandidateUser]] ): Seq[CandidateSource[T, CandidateUser]] = { val typeToFilter = request.params(GlobalParams.CandidateSourcesToFilter) val sourcesToFilter = CandidateSourceTypeToMap.get(typeToFilter).getOrElse(Set.empty) sources.filterNot { source => sourcesToFilter.contains(source.identifier) } } } object CandidateSourceHoldbackUtil { final val ContextualActivityCandidateSourceIds: Set[CandidateSourceIdentifier] = Set( RecentFollowingSimilarUsersSource.Identifier, RecentEngagementNonDirectFollowSource.Identifier, RecentEngagementSimilarUsersSource.Identifier, RecentStrongEngagementDirectFollowSimilarUsersSource.Identifier, SwitchingSimsSource.Identifier, ) final val SocialCandidateSourceIds: Set[CandidateSourceIdentifier] = Set( ForwardEmailBookSource.Identifier, ForwardPhoneBookSource.Identifier, ReverseEmailBookSource.Identifier, ReversePhoneBookSource.Identifier, RecentFollowingRecentFollowingExpansionSource.Identifier, BaseOnlineSTPSource.Identifier, MutualFollowStrongTiePredictionSource.Identifier, OfflineStrongTiePredictionSource.Identifier, SocialProofEnforcedOfflineStrongTiePredictionSource.Identifier, TriangularLoopsSource.Identifier, TwoHopRandomWalkSource.Identifier ) final val GeoCandidateSourceIds: Set[CandidateSourceIdentifier] = Set( PPMILocaleFollowSource.Identifier, PopCountrySource.Identifier, PopGeohashSource.Identifier, PopCountryBackFillSource.Identifier, PopGeoSource.Identifier, ) final val CandidateSourceTypeToMap: Map[CandidateSourceType.Value, Set[ CandidateSourceIdentifier ]] = Map( CandidateSourceType.Social -> SocialCandidateSourceIds, CandidateSourceType.ActivityContextual -> ContextualActivityCandidateSourceIds, CandidateSourceType.GeoAndInterests -> GeoCandidateSourceIds ) } ================================================ FILE: follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/RecommendationFlowBaseSideEffectsUtil.scala ================================================ package com.twitter.follow_recommendations.utils import com.twitter.follow_recommendations.common.base.RecommendationFlow import com.twitter.follow_recommendations.common.base.SideEffectsUtil import com.twitter.follow_recommendations.common.models.CandidateUser import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch trait RecommendationFlowBaseSideEffectsUtil[Target <: HasClientContext, Candidate <: CandidateUser] extends SideEffectsUtil[Target, Candidate] { recommendationFlow: RecommendationFlow[Target, Candidate] => override def applySideEffects( target: Target, candidateSources: Seq[CandidateSource[Target, Candidate]], candidatesFromCandidateSources: Seq[Candidate], mergedCandidates: Seq[Candidate], filteredCandidates: Seq[Candidate], rankedCandidates: Seq[Candidate], transformedCandidates: Seq[Candidate], truncatedCandidates: Seq[Candidate], results: Seq[Candidate] ): Stitch[Unit] = { Stitch.async( Stitch.collect( Seq( applySideEffectsCandidateSourceCandidates( target, candidateSources, candidatesFromCandidateSources), applySideEffectsMergedCandidates(target, mergedCandidates), applySideEffectsFilteredCandidates(target, filteredCandidates), applySideEffectsRankedCandidates(target, rankedCandidates), applySideEffectsTransformedCandidates(target, transformedCandidates), applySideEffectsTruncatedCandidates(target, truncatedCandidates), applySideEffectsResults(target, results) ) )) } /* In subclasses, override functions below to apply custom side effects at each step in pipeline. Call super.applySideEffectsXYZ to scribe basic scribes implemented in this parent class */ def applySideEffectsCandidateSourceCandidates( target: Target, candidateSources: Seq[CandidateSource[Target, Candidate]], candidatesFromCandidateSources: Seq[Candidate] ): Stitch[Unit] = { val candidatesGroupedByCandidateSources = candidatesFromCandidateSources.groupBy( _.getPrimaryCandidateSource.getOrElse(CandidateSourceIdentifier("NoCandidateSource"))) target.getOptionalUserId match { case Some(userId) => val userAgeOpt = SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays) userAgeOpt match { case Some(userAge) if userAge <= 30 => candidateSources.map { candidateSource => { val candidateSourceStats = statsReceiver.scope(candidateSource.identifier.name) val isEmpty = !candidatesGroupedByCandidateSources.keySet.contains(candidateSource.identifier) if (userAge <= 1) candidateSourceStats .scope("user_age", "1", "empty").counter(isEmpty.toString).incr() if (userAge <= 7) candidateSourceStats .scope("user_age", "7", "empty").counter(isEmpty.toString).incr() if (userAge <= 30) candidateSourceStats .scope("user_age", "30", "empty").counter(isEmpty.toString).incr() } } case _ => Nil } case None => Nil } Stitch.Unit } def applySideEffectsBaseCandidates( target: Target, candidates: Seq[Candidate] ): Stitch[Unit] = Stitch.Unit def applySideEffectsMergedCandidates( target: Target, candidates: Seq[Candidate] ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) def applySideEffectsFilteredCandidates( target: Target, candidates: Seq[Candidate] ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) def applySideEffectsRankedCandidates( target: Target, candidates: Seq[Candidate] ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) def applySideEffectsTransformedCandidates( target: Target, candidates: Seq[Candidate] ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) def applySideEffectsTruncatedCandidates( target: Target, candidates: Seq[Candidate] ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) def applySideEffectsResults( target: Target, candidates: Seq[Candidate] ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) } ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/BUILD ================================================ create_thrift_libraries( base_name = "thrift", sources = ["*.thrift"], platform = "java8", tags = ["bazel-compatible"], dependency_roots = [ "finatra-internal/thrift/src/main/thrift", "follow-recommendations-service/thrift/src/main/thrift/logging:thrift", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift", "src/thrift/com/twitter/ads/adserver:adserver_common", "src/thrift/com/twitter/ml/api:data", "src/thrift/com/twitter/suggests/controller_data", ], generate_languages = [ "java", "scala", "strato", ], provides_java_name = "follow-recommendations-java", provides_scala_name = "follow-recommendations-scala", ) ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/assembler.thrift ================================================ namespace java com.twitter.follow_recommendations.thriftjava #@namespace scala com.twitter.follow_recommendations.thriftscala #@namespace strato com.twitter.follow_recommendations struct Header { 1: required Title title } struct Title { 1: required string text } struct Footer { 1: optional Action action } struct Action { 1: required string text 2: required string actionURL } struct UserList { 1: required bool userBioEnabled 2: required bool userBioTruncated 3: optional i64 userBioMaxLines 4: optional FeedbackAction feedbackAction } struct Carousel { 1: optional FeedbackAction feedbackAction } union WTFPresentation { 1: UserList userBioList 2: Carousel carousel } struct DismissUserId {} union FeedbackAction { 1: DismissUserId dismissUserId } ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/client_context.thrift ================================================ namespace java com.twitter.follow_recommendations.thriftjava #@namespace scala com.twitter.follow_recommendations.thriftscala #@namespace strato com.twitter.follow_recommendations // Caller/Client level specific context (e.g, user id/guest id/app id). struct ClientContext { 1: optional i64 userId(personalDataType='UserId') 2: optional i64 guestId(personalDataType='GuestId') 3: optional i64 appId(personalDataType='AppId') 4: optional string ipAddress(personalDataType='IpAddress') 5: optional string userAgent(personalDataType='UserAgent') 6: optional string countryCode(personalDataType='InferredCountry') 7: optional string languageCode(personalDataType='InferredLanguage') 9: optional bool isTwoffice(personalDataType='InferredLocation') 10: optional set userRoles 11: optional string deviceId(personalDataType='DeviceId') 12: optional i64 guestIdAds(personalDataType='GuestId') 13: optional i64 guestIdMarketing(personalDataType='GuestId') }(hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/debug.thrift ================================================ namespace java com.twitter.follow_recommendations.thriftjava #@namespace scala com.twitter.follow_recommendations.thriftscala #@namespace strato com.twitter.follow_recommendation // These are broken into their own union // because we can have features that are // complex flavors of these (such as Seq) union PrimitiveFeatureValue { 1: i32 intValue 2: i64 longValue 3: string strValue 4: bool boolValue } union FeatureValue { 1: PrimitiveFeatureValue primitiveValue } struct DebugParams { 1: optional map featureOverrides 2: optional i64 randomizationSeed 3: optional bool includeDebugInfoInResults 4: optional bool doNotLog } enum DebugCandidateSourceIdentifier { UTT_INTERESTS_RELATED_USERS_SOURCE = 0 UTT_PRODUCER_EXPANSION_SOURCE = 1 UTT_SEED_ACCOUNT_SOURCE = 2 BYF_USER_FOLLOW_CLUSTER_SIMS_SOURCE = 3 BYF_USER_FOLLOW_CLUSTER_SOURCE = 4 USER_FOLLOW_CLUSTER_SOURCE = 5 RECENT_SEARCH_BASED_SOURCE = 6 PEOPLE_ACTIVITY_RECENT_ENGAGEMENT_SOURCE = 7 PEOPLE_ACTIVITY_RECENT_ENGAGEMENT_SIMS_SOURCE = 8, REVERSE_PHONE_BOOK_SOURCE = 9, REVERSE_EMAIL_BOOK_SOURCE = 10, SIMS_DEBUG_STORE = 11, UTT_PRODUCER_ONLINE_MBCG_SOURCE = 12, BONUS_FOLLOW_CONDITIONAL_ENGAGEMENT_STORE = 13, // 14 (BONUS_FOLLOW_PMI_STORE) was deleted as it's not used anymore FOLLOW2VEC_NEAREST_NEIGHBORS_STORE = 15, OFFLINE_STP = 16, OFFLINE_STP_BIG = 17, OFFLINE_MUTUAL_FOLLOW_EXPANSION = 18, REPEATED_PROFILE_VISITS = 19, TIME_DECAY_FOLLOW2VEC_NEAREST_NEIGHBORS_STORE = 20, LINEAR_REGRESSION_FOLLOW2VEC_NEAREST_NEIGHBORS_STORE = 21, REAL_GRAPH_EXPANSION_SOURCE = 22, RELATABLE_ACCOUNTS_BY_INTEREST = 23, EMAIL_TWEET_CLICK = 24, GOOD_TWEET_CLICK_ENGAGEMENTS = 25, ENGAGED_FOLLOWER_RATIO = 26, TWEET_SHARE_ENGAGEMENTS = 27, BULK_FRIEND_FOLLOWS = 28, REAL_GRAPH_OON_V2_SOURCE = 30, CROWD_SEARCH_ACCOUNTS = 31, POP_GEOHASH = 32, POP_COUNTRY = 33, POP_COUNTRY_BACKFILL = 34, TWEET_SHARER_TO_SHARE_RECIPIENT_ENGAGEMENTS = 35, TWEET_AUTHOR_TO_SHARE_RECIPIENT_ENGAGEMENTS = 36, BULK_FRIEND_FOLLOWS_NEW_USER = 37, ONLINE_STP_EPSCORER = 38, ORGANIC_FOLLOW_ACCOUNTS = 39, NUX_LO_HISTORY = 40, TRAFFIC_ATTRIBUTION_ACCOUNTS = 41, ONLINE_STP_RAW_ADDRESS_BOOK = 42, POP_GEOHASH_QUALITY_FOLLOW = 43, NOTIFICATION_ENGAGEMENT = 44, EFR_BY_WORLDWIDE_PICTURE_PRODUCER = 45, POP_GEOHASH_REAL_GRAPH = 46, } ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/display_context.thrift ================================================ include "flows.thrift" include "recently_engaged_user_id.thrift" namespace java com.twitter.follow_recommendations.thriftjava #@namespace scala com.twitter.follow_recommendations.thriftscala #@namespace strato com.twitter.follow_recommendations struct Profile { 1: required i64 profileId(personalDataType='UserId') }(hasPersonalData='true') struct Search { 1: required string searchQuery(personalDataType='SearchQuery') }(hasPersonalData='true') struct Rux { 1: required i64 focalAuthorId(personalDataType='UserId') }(hasPersonalData='true') struct Topic { 1: required i64 topicId(personalDataType = 'TopicFollow') }(hasPersonalData='true') struct ReactiveFollow { 1: required list followedUserIds(personalDataType='UserId') }(hasPersonalData='true') struct NuxInterests { 1: optional flows.FlowContext flowContext // set for recommendation inside an interactive flow 2: optional list uttInterestIds // if provided, we use these interestIds for generating candidates instead of for example fetching user selected interests }(hasPersonalData='true') struct AdCampaignTarget { 1: required list similarToUserIds(personalDataType='UserId') }(hasPersonalData='true') struct ConnectTab { 1: required list byfSeedUserIds(personalDataType='UserId') 2: required list similarToUserIds(personalDataType='UserId') 3: required list recentlyEngagedUserIds }(hasPersonalData='true') struct SimilarToUser { 1: required i64 similarToUserId(personalDataType='UserId') }(hasPersonalData='true') struct PostNuxFollowTask { 1: optional flows.FlowContext flowContext // set for recommendation inside an interactive flow }(hasPersonalData='true') union DisplayContext { 1: Profile profile 2: Search search 3: Rux rux 4: Topic topic 5: ReactiveFollow reactiveFollow 6: NuxInterests nuxInterests 7: AdCampaignTarget adCampaignTarget 8: ConnectTab connectTab 9: SimilarToUser similarToUser 10: PostNuxFollowTask postNuxFollowTask }(hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/display_location.thrift ================================================ namespace java com.twitter.follow_recommendations.thriftjava #@namespace scala com.twitter.follow_recommendations.thriftscala #@namespace strato com.twitter.follow_recommendations enum DisplayLocation { SIDEBAR = 0 PROFILE_SIDEBAR = 2 CLUSTER_FOLLOW = 7 NEW_USER_SARUS_BACKFILL = 12 PROFILE_DEVICE_FOLLOW = 23 RECOS_BACKFILL = 32 HOME_TIMELINE = 39 # HOME_TIMELINE_WTF in Hermit PROFILE_TOP_FOLLOWING = 42 PROFILE_TOP_FOLLOWERS = 43 PEOPLE_PLUS_PLUS = 47 EXPLORE_TAB = 57 MagicRecs = 59 # Account recommendation in notification AB_UPLOAD_INJECTION = 60 /** * To prevent setting 2 display locations with the same index in FRS. * * The display location should be added to the following files: * - follow-recommendations-service/thrift/src/main/thrift/display_location.thrift * - follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift * - follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala */ CAMPAIGN_FORM = 61 RUX_LANDING_PAGE = 62 PROFILE_BONUS_FOLLOW = 63 ELECTION_EXPLORE_WTF = 64 HTL_BONUS_FOLLOW = 65 TOPIC_LANDING_PAGE_HEADER = 66 NUX_PYMK = 67 NUX_INTERESTS = 68 REACTIVE_FOLLOW = 69 RUX_PYMK = 70 INDIA_COVID19_CURATED_ACCOUNTS_WTF = 71 NUX_TOPIC_BONUS_FOLLOW = 72 TWEET_NOTIFICATION_RECS = 73 HTL_SPACE_HOSTS = 74 POST_NUX_FOLLOW_TASK = 75 TOPIC_LANDING_PAGE = 76 USER_TYPEAHEAD_PREFETCH = 77 HOME_TIMELINE_RELATABLE_ACCOUNTS = 78 NUX_GEO_CATEGORY = 79 NUX_INTERESTS_CATEGORY = 80 NUX_PYMK_CATEGORY = 81 TOP_ARTICLES = 82 HOME_TIMELINE_TWEET_RECS = 83 HTL_BULK_FRIEND_FOLLOWS = 84 NUX_AUTO_FOLLOW = 85 SEARCH_BONUS_FOLLOW = 86 CONTENT_RECOMMENDER = 87 HOME_TIMELINE_REVERSE_CHRON = 88 } ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/engagementType.thrift ================================================ namespace java com.twitter.follow_recommendations.thriftjava #@namespace scala com.twitter.follow_recommendations.thriftscala #@namespace strato com.twitter.follow_recommendations enum EngagementType { Click = 0 Like = 1 Mention = 2 Retweet = 3 ProfileView = 4 } ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/flows.thrift ================================================ /* * This file defines additional thrift objects that should be specified in FRS request for context of recommendation, specifically the previous recommendations / new interactions in an interactive flow (series of follow steps). These typically are sent from OCF */ namespace java com.twitter.follow_recommendations.thriftjava #@namespace scala com.twitter.follow_recommendations.thriftscala #@namespace strato com.twitter.follow_recommendations struct FlowRecommendation { 1: required i64 userId(personalDataType='UserId') }(hasPersonalData='true') struct RecommendationStep { 1: required list recommendations 2: required set followedUserIds(personalDataType='UserId') }(hasPersonalData='true') struct FlowContext { 1: required list steps }(hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/follow-recommendations-service.thrift ================================================ namespace java com.twitter.follow_recommendations.thriftjava #@namespace scala com.twitter.follow_recommendations.thriftscala #@namespace strato com.twitter.follow_recommendations include "assembler.thrift" include "client_context.thrift" include "debug.thrift" include "display_context.thrift" include "display_location.thrift" include "recommendations.thrift" include "recently_engaged_user_id.thrift" include "finatra-thrift/finatra_thrift_exceptions.thrift" include "com/twitter/product_mixer/core/pipeline_execution_result.thrift" struct RecommendationRequest { 1: required client_context.ClientContext clientContext 2: required display_location.DisplayLocation displayLocation 3: optional display_context.DisplayContext displayContext // Max results to return 4: optional i32 maxResults // Cursor to continue returning results if any 5: optional string cursor // IDs of Content to exclude from recommendations 6: optional list excludedIds(personalDataType='UserId') // Whether to also get promoted content 7: optional bool fetchPromotedContent 8: optional debug.DebugParams debugParams 9: optional string userLocationState(personalDataType='InferredLocation') }(hasPersonalData='true') struct RecommendationResponse { 1: required list recommendations }(hasPersonalData='true') // for scoring a list of candidates, while logging hydrated features struct ScoringUserRequest { 1: required client_context.ClientContext clientContext 2: required display_location.DisplayLocation displayLocation 3: required list candidates 4: optional debug.DebugParams debugParams }(hasPersonalData='true') struct ScoringUserResponse { 1: required list candidates // empty for now }(hasPersonalData='true') // for getting the list of candidates generated by a single candidate source struct DebugCandidateSourceRequest { 1: required client_context.ClientContext clientContext 2: required debug.DebugCandidateSourceIdentifier candidateSource 3: optional list uttInterestIds 4: optional debug.DebugParams debugParams 5: optional list recentlyFollowedUserIds 6: optional list recentlyEngagedUserIds 7: optional list byfSeedUserIds 8: optional list similarToUserIds 9: required bool applySgsPredicate 10: optional i32 maxResults }(hasPersonalData='true') service FollowRecommendationsThriftService { RecommendationResponse getRecommendations(1: RecommendationRequest request) throws ( 1: finatra_thrift_exceptions.ServerError serverError, 2: finatra_thrift_exceptions.UnknownClientIdError unknownClientIdError, 3: finatra_thrift_exceptions.NoClientIdError noClientIdError ) RecommendationDisplayResponse getRecommendationDisplayResponse(1: RecommendationRequest request) throws ( 1: finatra_thrift_exceptions.ServerError serverError, 2: finatra_thrift_exceptions.UnknownClientIdError unknownClientIdError, 3: finatra_thrift_exceptions.NoClientIdError noClientIdError ) // temporary endpoint for feature hydration and logging for data collection. ScoringUserResponse scoreUserCandidates(1: ScoringUserRequest request) throws ( 1: finatra_thrift_exceptions.ServerError serverError, 2: finatra_thrift_exceptions.UnknownClientIdError unknownClientIdError, 3: finatra_thrift_exceptions.NoClientIdError noClientIdError ) // Debug endpoint for getting recommendations of a single candidate source. We can remove this endpoint when ProMix provide this functionality and we integrate with it. RecommendationResponse debugCandidateSource(1: DebugCandidateSourceRequest request) throws ( 1: finatra_thrift_exceptions.ServerError serverError, 2: finatra_thrift_exceptions.UnknownClientIdError unknownClientIdError, 3: finatra_thrift_exceptions.NoClientIdError noClientIdError ) // Get the full execution log for a pipeline (used by our debugging tools) pipeline_execution_result.PipelineExecutionResult executePipeline(1: RecommendationRequest request) throws ( 1: finatra_thrift_exceptions.ServerError serverError, 2: finatra_thrift_exceptions.UnknownClientIdError unknownClientIdError, 3: finatra_thrift_exceptions.NoClientIdError noClientIdError ) } struct RecommendationDisplayResponse { 1: required list hydratedRecommendation 2: optional assembler.Header header 3: optional assembler.Footer footer 4: optional assembler.WTFPresentation wtfPresentation }(hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/follow_recommendations_serving_history.thrift ================================================ namespace java com.twitter.follow_recommendations.thriftjava #@namespace scala com.twitter.follow_recommendations.thriftscala #@namespace strato com.twitter.follow_recommendations // struct used for storing the history of computing and serving of recommendations to a user struct FollowRecommendationsServingHistory { 1: required i64 lastComputationTimeMs (personalDataType = 'PrivateTimestamp') 2: required i64 lastServingTimeMs (personalDataType = 'PrivateTimestamp') }(persisted='true', hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/logging/BUILD ================================================ create_thrift_libraries( base_name = "thrift", sources = ["*.thrift"], platform = "java8", tags = ["bazel-compatible"], dependency_roots = [ "src/thrift/com/twitter/ads/adserver:adserver_common", "src/thrift/com/twitter/ml/api:data", "src/thrift/com/twitter/suggests/controller_data", ], generate_languages = [ "java", "scala", "strato", ], provides_java_name = "follow-recommendations-logging-java", provides_scala_name = "follow-recommendations-logging-scala", ) ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/logging/client_context.thrift ================================================ namespace java com.twitter.follow_recommendations.logging.thriftjava #@namespace scala com.twitter.follow_recommendations.logging.thriftscala #@namespace strato com.twitter.follow_recommendations.logging // Offline equal of ClientContext struct OfflineClientContext { 1: optional i64 userId(personalDataType='UserId') 2: optional i64 guestId(personalDataType='GuestId') 3: optional i64 appId(personalDataType='AppId') 4: optional string countryCode(personalDataType='InferredCountry') 5: optional string languageCode(personalDataType='InferredLanguage') 6: optional i64 guestIdAds(personalDataType='GuestId') 7: optional i64 guestIdMarketing(personalDataType='GuestId') }(persisted='true', hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/logging/debug.thrift ================================================ namespace java com.twitter.follow_recommendations.logging.thriftjava #@namespace scala com.twitter.follow_recommendations.logging.thriftscala #@namespace strato com.twitter.follow_recommendation.logging // subset of DebugParams struct OfflineDebugParams { 1: optional i64 randomizationSeed // track if the request was randomly ranked or not }(persisted='true', hasPersonalData='false') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/logging/display_context.thrift ================================================ include "logging/flows.thrift" include "logging/recently_engaged_user_id.thrift" namespace java com.twitter.follow_recommendations.logging.thriftjava #@namespace scala com.twitter.follow_recommendations.logging.thriftscala #@namespace strato com.twitter.follow_recommendations.logging // Offline equal of Profile DisplayContext struct OfflineProfile { 1: required i64 profileId(personalDataType='UserId') }(persisted='true', hasPersonalData='true') // Offline equal of Search DisplayContext struct OfflineSearch { 1: required string searchQuery(personalDataType='SearchQuery') }(persisted='true', hasPersonalData='true') // Offline equal of Rux Landing Page DisplayContext struct OfflineRux { 1: required i64 focalAuthorId(personalDataType="UserId") }(persisted='true', hasPersonalData='true') // Offline equal of Topic DisplayContext struct OfflineTopic { 1: required i64 topicId(personalDataType = 'TopicFollow') }(persisted='true', hasPersonalData='true') struct OfflineReactiveFollow { 1: required list followedUserIds(personalDataType='UserId') }(persisted='true', hasPersonalData='true') struct OfflineNuxInterests { 1: optional flows.OfflineFlowContext flowContext // set for recommendation inside an interactive flow }(persisted='true', hasPersonalData='true') struct OfflineAdCampaignTarget { 1: required list similarToUserIds(personalDataType='UserId') }(persisted='true', hasPersonalData='true') struct OfflineConnectTab { 1: required list byfSeedUserIds(personalDataType='UserId') 2: required list similarToUserIds(personalDataType='UserId') 3: required list recentlyEngagedUserIds }(persisted='true', hasPersonalData='true') struct OfflineSimilarToUser { 1: required i64 similarToUserId(personalDataType='UserId') }(persisted='true', hasPersonalData='true') struct OfflinePostNuxFollowTask { 1: optional flows.OfflineFlowContext flowContext // set for recommendation inside an interactive flow }(persisted='true', hasPersonalData='true') // Offline equal of DisplayContext union OfflineDisplayContext { 1: OfflineProfile profile 2: OfflineSearch search 3: OfflineRux rux 4: OfflineTopic topic 5: OfflineReactiveFollow reactiveFollow 6: OfflineNuxInterests nuxInterests 7: OfflineAdCampaignTarget adCampaignTarget 8: OfflineConnectTab connectTab 9: OfflineSimilarToUser similarToUser 10: OfflinePostNuxFollowTask postNuxFollowTask }(persisted='true', hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift ================================================ namespace java com.twitter.follow_recommendations.logging.thriftjava #@namespace scala com.twitter.follow_recommendations.logging.thriftscala #@namespace strato com.twitter.follow_recommendations.logging /** * Make sure you add the new DL to the following files and redeploy our attribution jobs * - follow-recommendations-service/thrift/src/main/thrift/display_location.thrift * - follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift * - follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala */ // Offline equal of DisplayLocation enum OfflineDisplayLocation { SIDEBAR = 0 PROFILE_SIDEBAR = 2 CLUSTER_FOLLOW = 7 NEW_USER_SARUS_BACKFILL = 12 PROFILE_DEVICE_FOLLOW = 23 RECOS_BACKFILL = 32 HOME_TIMELINE = 39 PROFILE_TOP_FOLLOWING = 42 PROFILE_TOP_FOLLOWERS = 43 PEOPLE_PLUS_PLUS = 47 EXPLORE_TAB = 57 MagicRecs = 59 AB_UPLOAD_INJECTION = 60 CAMPAIGN_FORM = 61 RUX_LANDING_PAGE = 62 PROFILE_BONUS_FOLLOW = 63 ELECTION_EXPLORE_WTF = 64 HTL_BONUS_FOLLOW = 65 TOPIC_LANDING_PAGE_HEADER = 66 NUX_PYMK = 67 NUX_INTERESTS = 68 REACTIVE_FOLLOW = 69 RUX_PYMK = 70 INDIA_COVID19_CURATED_ACCOUNTS_WTF=71 NUX_TOPIC_BONUS_FOLLOW = 72 TWEET_NOTIFICATION_RECS = 73 HTL_SPACE_HOSTS = 74 POST_NUX_FOLLOW_TASK = 75 TOPIC_LANDING_PAGE = 76 USER_TYPEAHEAD_PREFETCH = 77 HOME_TIMELINE_RELATABLE_ACCOUNTS = 78 NUX_GEO_CATEGORY = 79 NUX_INTERESTS_CATEGORY = 80 NUX_PYMK_CATEGORY = 81 TOP_ARTICLES = 82 HOME_TIMELINE_TWEET_RECS = 83 HTL_BULK_FRIEND_FOLLOWS = 84 NUX_AUTO_FOLLOW = 85 SEARCH_BONUS_FOLLOW = 86 CONTENT_RECOMMENDER = 87 HOME_TIMELINE_REVERSE_CHRON = 88 }(persisted='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/logging/engagementType.thrift ================================================ namespace java com.twitter.follow_recommendations.logging.thriftjava #@namespace scala com.twitter.follow_recommendations.logging.thriftscala #@namespace strato com.twitter.follow_recommendations.logging enum EngagementType { Click = 0 Like = 1 Mention = 2 Retweet = 3 ProfileView = 4 } ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/logging/flows.thrift ================================================ namespace java com.twitter.follow_recommendations.logging.thriftjava #@namespace scala com.twitter.follow_recommendations.logging.thriftscala #@namespace strato com.twitter.follow_recommendations.logging struct OfflineFlowRecommendation { 1: required i64 userId(personalDataType='UserId') }(persisted='true', hasPersonalData='true') struct OfflineRecommendationStep { 1: required list recommendations 2: required set followedUserIds(personalDataType='UserId') }(persisted='true', hasPersonalData='true') struct OfflineFlowContext { 1: required list steps }(persisted='true', hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/logging/logs.thrift ================================================ namespace java com.twitter.follow_recommendations.logging.thriftjava #@namespace scala com.twitter.follow_recommendations.logging.thriftscala #@namespace strato com.twitter.follow_recommendations.logging include "client_context.thrift" include "debug.thrift" include "display_context.thrift" include "display_location.thrift" include "recommendations.thrift" struct OfflineRecommendationRequest { 1: required client_context.OfflineClientContext clientContext 2: required display_location.OfflineDisplayLocation displayLocation 3: optional display_context.OfflineDisplayContext displayContext 4: optional i32 maxResults 5: optional string cursor 6: optional list excludedIds(personalDataType='UserId') 7: optional bool fetchPromotedContent 8: optional debug.OfflineDebugParams debugParams }(persisted='true', hasPersonalData='true') struct OfflineRecommendationResponse { 1: required list recommendations }(persisted='true', hasPersonalData='true') struct RecommendationLog { 1: required OfflineRecommendationRequest request 2: required OfflineRecommendationResponse response 3: required i64 timestampMs }(persisted='true', hasPersonalData='true') struct OfflineScoringUserRequest { 1: required client_context.OfflineClientContext clientContext 2: required display_location.OfflineDisplayLocation displayLocation 3: required list candidates }(persisted='true', hasPersonalData='true') struct OfflineScoringUserResponse { 1: required list candidates }(persisted='true', hasPersonalData='true') struct ScoredUsersLog { 1: required OfflineScoringUserRequest request 2: required OfflineScoringUserResponse response 3: required i64 timestampMs }(persisted='true', hasPersonalData='true') struct OfflineRecommendationFlowUserMetadata { 1: optional i32 userSignupAge(personalDataType = 'AgeOfAccount') 2: optional string userState(personalDataType = 'UserState') }(persisted='true', hasPersonalData='true') struct OfflineRecommendationFlowSignals { 1: optional string countryCode(personalDataType='InferredCountry') }(persisted='true', hasPersonalData='true') struct OfflineRecommendationFlowCandidateSourceCandidates { 1: required string candidateSourceName 2: required list candidateUserIds(personalDataType='UserId') 3: optional list candidateUserScores }(persisted='true', hasPersonalData='true') struct RecommendationFlowLog { 1: required client_context.OfflineClientContext clientContext 2: optional OfflineRecommendationFlowUserMetadata userMetadata 3: optional OfflineRecommendationFlowSignals signals 4: required i64 timestampMs 5: required string recommendationFlowIdentifier 6: optional list filteredCandidates 7: optional list rankedCandidates 8: optional list truncatedCandidates }(persisted='true', hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/logging/reasons.thrift ================================================ namespace java com.twitter.follow_recommendations.logging.thriftjava #@namespace scala com.twitter.follow_recommendations.logging.thriftscala #@namespace strato com.twitter.follow_recommendations.logging // Proof based on Follow relationship struct FollowProof { 1: required list userIds(personalDataType='UserId') 2: required i32 numIds(personalDataType='CountOfFollowersAndFollowees') }(persisted='true', hasPersonalData='true') // Similar to userIds in the context (e.g. profileId) struct SimilarToProof { 1: required list userIds(personalDataType='UserId') }(persisted='true', hasPersonalData='true') // Proof based on geo location struct PopularInGeoProof { 1: required string location(personalDataType='InferredLocation') }(persisted='true', hasPersonalData='true') // Proof based on ttt interest struct TttInterestProof { 1: required i64 interestId(personalDataType='ProvidedInterests') 2: required string interestDisplayName(personalDataType='ProvidedInterests') }(persisted='true', hasPersonalData='true') // Proof based on topics struct TopicProof { 1: required i64 topicId(personalDataType='ProvidedInterests') }(persisted='true', hasPersonalData='true') // Proof based on custom interest / search queries struct CustomInterestProof { 1: required string customerInterest(personalDataType='SearchQuery') }(persisted='true', hasPersonalData='true') // Proof based on tweet authors struct TweetsAuthorProof { 1: required list tweetIds(personalDataType='TweetId') }(persisted='true', hasPersonalData='true') // Proof candidate is of device follow type struct DeviceFollowProof { 1: required bool isDeviceFollow(personalDataType='OtherDeviceInfo') }(persisted='true', hasPersonalData='true') // Account level proof that should be attached to each candidate struct AccountProof { 1: optional FollowProof followProof 2: optional SimilarToProof similarToProof 3: optional PopularInGeoProof popularInGeoProof 4: optional TttInterestProof tttInterestProof 5: optional TopicProof topicProof 6: optional CustomInterestProof customInterestProof 7: optional TweetsAuthorProof tweetsAuthorProof 8: optional DeviceFollowProof deviceFollowProof }(persisted='true', hasPersonalData='true') struct Reason { 1: optional AccountProof accountProof }(persisted='true', hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/logging/recently_engaged_user_id.thrift ================================================ namespace java com.twitter.follow_recommendations.logging.thriftjava #@namespace scala com.twitter.follow_recommendations.logging.thriftscala #@namespace strato com.twitter.follow_recommendations.logging include "engagementType.thrift" struct RecentlyEngagedUserId { 1: required i64 id(personalDataType='UserId') 2: required engagementType.EngagementType engagementType }(persisted='true', hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/logging/recommendations.thrift ================================================ namespace java com.twitter.follow_recommendations.logging.thriftjava #@namespace scala com.twitter.follow_recommendations.logging.thriftscala #@namespace strato com.twitter.follow_recommendations.logging include "com/twitter/ads/adserver/adserver_common.thrift" include "reasons.thrift" include "tracking.thrift" include "scoring.thrift" // Offline equal of UserRecommendation struct OfflineUserRecommendation { 1: required i64 userId(personalDataType='UserId') // reason for this suggestions, eg: social context 2: optional reasons.Reason reason // present if it is a promoted account 3: optional adserver_common.AdImpression adImpression // tracking token (unserialized) for attribution 4: optional tracking.TrackingToken trackingToken // scoring details 5: optional scoring.ScoringDetails scoringDetails }(persisted='true', hasPersonalData='true') // Offline equal of Recommendation union OfflineRecommendation { 1: OfflineUserRecommendation user }(persisted='true', hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/logging/scoring.thrift ================================================ namespace java com.twitter.follow_recommendations.logging.thriftjava #@namespace scala com.twitter.follow_recommendations.logging.thriftscala #@namespace strato com.twitter.follow_recommendations.logging include "com/twitter/ml/api/data.thrift" struct CandidateSourceDetails { 1: optional map candidateSourceScores 2: optional i32 primarySource }(persisted='true', hasPersonalData='false') struct Score { 1: required double value 2: optional string rankerId 3: optional string scoreType }(persisted='true', hasPersonalData='false') // scoring and ranking info per ranking stage // Contains (1) the ML-based heavy ranker and score (2) scores and rankers in producer experiment framework struct Scores { 1: required list scores 2: optional string selectedRankerId 3: required bool isInProducerScoringExperiment }(persisted='true', hasPersonalData='false') struct RankingInfo { 1: optional Scores scores 2: optional i32 rank }(persisted='true', hasPersonalData='false') // this encapsulates all information related to the ranking process from generation to scoring struct ScoringDetails { 1: optional CandidateSourceDetails candidateSourceDetails 2: optional double score // The ML-based heavy ranker score 3: optional data.DataRecord dataRecord 4: optional list rankerIds // all ranker ids, including (1) ML-based heavy ranker (2) non-ML adhoc rankers 5: optional map infoPerRankingStage // scoring and ranking info per ranking stage }(persisted='true', hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/logging/tracking.thrift ================================================ namespace java com.twitter.follow_recommendations.logging.thriftjava #@namespace scala com.twitter.follow_recommendations.logging.thriftscala #@namespace strato com.twitter.follow_recommendations.logging include "com/twitter/suggests/controller_data/controller_data.thrift" include "display_location.thrift" struct TrackingToken { // trace-id of the request 1: required i64 sessionId (personalDataType='SessionId') 2: optional display_location.OfflineDisplayLocation displayLocation // 64-bit encoded binary attributes of our recommendation 3: optional controller_data.ControllerData controllerData // WTF Algorithm Id (backward compatibility) 4: optional i32 algoId }(persisted='true', hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/reasons.thrift ================================================ namespace java com.twitter.follow_recommendations.thriftjava #@namespace scala com.twitter.follow_recommendations.thriftscala #@namespace strato com.twitter.follow_recommendations // Proof based on Follow relationship struct FollowProof { 1: required list userIds(personalDataType='UserId') 2: required i32 numIds(personalDataType='CountOfFollowersAndFollowees') }(hasPersonalData='true') // Similar to userIds in the context (e.g. profileId) struct SimilarToProof { 1: required list userIds(personalDataType='UserId') }(hasPersonalData='true') // Proof based on geo location struct PopularInGeoProof { 1: required string location(personalDataType='InferredLocation') }(hasPersonalData='true') // Proof based on ttt interest struct TttInterestProof { 1: required i64 interestId(personalDataType='ProvidedInterests') 2: required string interestDisplayName(personalDataType='ProvidedInterests') }(hasPersonalData='true') // Proof based on topics struct TopicProof { 1: required i64 topicId(personalDataType='ProvidedInterests') }(hasPersonalData='true') // Proof based on custom interest / search queries struct CustomInterestProof { 1: required string query(personalDataType='SearchQuery') }(hasPersonalData='true') // Proof based on tweet authors struct TweetsAuthorProof { 1: required list tweetIds(personalDataType='TweetId') }(hasPersonalData='true') // Proof candidate is of device follow type struct DeviceFollowProof { 1: required bool isDeviceFollow(personalDataType='OtherDeviceInfo') }(hasPersonalData='true') // Account level proof that should be attached to each candidate struct AccountProof { 1: optional FollowProof followProof 2: optional SimilarToProof similarToProof 3: optional PopularInGeoProof popularInGeoProof 4: optional TttInterestProof tttInterestProof 5: optional TopicProof topicProof 6: optional CustomInterestProof customInterestProof 7: optional TweetsAuthorProof tweetsAuthorProof 8: optional DeviceFollowProof deviceFollowProof }(hasPersonalData='true') struct Reason { 1: optional AccountProof accountProof }(hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/recently_engaged_user_id.thrift ================================================ namespace java com.twitter.follow_recommendations.thriftjava #@namespace scala com.twitter.follow_recommendations.thriftscala #@namespace strato com.twitter.follow_recommendations include "engagementType.thrift" struct RecentlyEngagedUserId { 1: required i64 id(personalDataType='UserId') 2: required engagementType.EngagementType engagementType }(persisted='true', hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/recommendations.thrift ================================================ namespace java com.twitter.follow_recommendations.thriftjava #@namespace scala com.twitter.follow_recommendations.thriftscala #@namespace strato com.twitter.follow_recommendations include "com/twitter/ads/adserver/adserver_common.thrift" include "debug.thrift" include "reasons.thrift" include "scoring.thrift" struct UserRecommendation { 1: required i64 userId(personalDataType='UserId') // reason for this suggestions, eg: social context 2: optional reasons.Reason reason // present if it is a promoted account 3: optional adserver_common.AdImpression adImpression // tracking token for attribution 4: optional string trackingInfo // scoring details 5: optional scoring.ScoringDetails scoringDetails 6: optional string recommendationFlowIdentifier // FeatureSwitch overrides for candidates: 7: optional map featureOverrides }(hasPersonalData='true') union Recommendation { 1: UserRecommendation user }(hasPersonalData='true') struct HydratedUserRecommendation { 1: required i64 userId(personalDataType='UserId') 2: optional string socialProof // present if it is a promoted account, used by clients for determining ad impression 3: optional adserver_common.AdImpression adImpression // tracking token for attribution 4: optional string trackingInfo }(hasPersonalData='true') union HydratedRecommendation { 1: HydratedUserRecommendation hydratedUserRecommendation } ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/scoring.thrift ================================================ namespace java com.twitter.follow_recommendations.thriftjava #@namespace scala com.twitter.follow_recommendations.thriftscala #@namespace strato com.twitter.follow_recommendations include "com/twitter/ml/api/data.thrift" struct CandidateSourceDetails { 1: optional map candidateSourceScores 2: optional i32 primarySource 3: optional map candidateSourceRanks }(hasPersonalData='false') struct Score { 1: required double value 2: optional string rankerId 3: optional string scoreType }(hasPersonalData='false') // Contains (1) the ML-based heavy ranker and score (2) scores and rankers in producer experiment framework struct Scores { 1: required list scores 2: optional string selectedRankerId 3: required bool isInProducerScoringExperiment }(hasPersonalData='false') struct RankingInfo { 1: optional Scores scores 2: optional i32 rank }(hasPersonalData='false') // this encapsulates all information related to the ranking process from generation to scoring struct ScoringDetails { 1: optional CandidateSourceDetails candidateSourceDetails 2: optional double score 3: optional data.DataRecord dataRecord 4: optional list rankerIds 5: optional DebugDataRecord debugDataRecord // this field is not logged as it's only used for debugging 6: optional map infoPerRankingStage // scoring and ranking info per ranking stage }(hasPersonalData='true') // exactly the same as a data record, except that we store the feature name instead of the id struct DebugDataRecord { 1: optional set binaryFeatures; // stores BINARY features 2: optional map continuousFeatures; // stores CONTINUOUS features 3: optional map discreteFeatures; // stores DISCRETE features 4: optional map stringFeatures; // stores STRING features 5: optional map> sparseBinaryFeatures; // stores sparse BINARY features 6: optional map> sparseContinuousFeatures; // sparse CONTINUOUS features }(hasPersonalData='true') ================================================ FILE: follow-recommendations-service/thrift/src/main/thrift/tracking.thrift ================================================ namespace java com.twitter.follow_recommendations.thriftjava #@namespace scala com.twitter.follow_recommendations.thriftscala #@namespace strato com.twitter.follow_recommendations include "com/twitter/suggests/controller_data/controller_data.thrift" include "display_location.thrift" // struct used for tracking/attribution purposes in our offline pipelines struct TrackingToken { // trace-id of the request 1: required i64 sessionId (personalDataType='SessionId') 2: optional display_location.DisplayLocation displayLocation // 64-bit encoded binary attributes of our recommendation 3: optional controller_data.ControllerData controllerData // WTF Algorithm Id (backward compatibility) 4: optional i32 algoId }(hasPersonalData='true') ================================================ FILE: graph-feature-service/BUILD.bazel ================================================ alias( name = "graph_feature_service-server", target = ":graph_feature_service-server_lib", ) target( name = "graph_feature_service-server_lib", dependencies = [ "graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server", ], ) alias( name = "graph_feature_service-worker", target = ":graph_feature_service-worker_lib", ) target( name = "graph_feature_service-worker_lib", dependencies = [ "graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker", ], ) jvm_binary( name = "server-bin", basename = "graph_feature_service-server", main = "com.twitter.graph_feature_service.server.Main", platform = "java8", tags = ["bazel-compatible"], dependencies = [ ":graph_feature_service-server", "3rdparty/jvm/ch/qos/logback:logback-classic", "finagle/finagle-zipkin-scribe/src/main/scala", "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", "twitter-server/logback-classic/src/main/scala", ], ) jvm_binary( name = "worker-bin", basename = "graph_feature_service-worker", main = "com.twitter.graph_feature_service.worker.Main", platform = "java8", tags = ["bazel-compatible"], dependencies = [ ":graph_feature_service-worker", "3rdparty/jvm/ch/qos/logback:logback-classic", "finagle/finagle-zipkin-scribe/src/main/scala", "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", "twitter-server/logback-classic/src/main/scala", ], ) jvm_app( name = "server-bundle", basename = "graph_feature_service-server-dist", binary = ":server-bin", tags = ["bazel-compatible"], ) jvm_app( name = "worker-bundle", basename = "graph_feature_service-worker-dist", binary = ":worker-bin", tags = ["bazel-compatible"], ) ================================================ FILE: graph-feature-service/README.md ================================================ # Graph Feature Service Graph Feature Service (GFS) is a distributed system that can provide various graph features for given pairs of users. For instance, given source user A and candidate user C, GFS can answer questions like “how many of A’s followings have favorited C”, “how many of A’s followings are following C”, and “how much C is similar to the users that A has favorited“. ================================================ FILE: graph-feature-service/doc/common.md ================================================ # Common thrift types GFS uses several thrift datastructures which are common to multiple queries. They are listed below. ## EdgeType `EdgeType` is a thrift enum which specifies which edge types to query for the graph. ```thrift enum EdgeType { FOLLOWING, FOLLOWED_BY, FAVORITE, FAVORITED_BY, RETWEET, RETWEETED_BY, REPLY, REPLYED_BY, MENTION, MENTIONED_BY, MUTUAL_FOLLOW, SIMILAR_TO, // more edge types (like block, report, etc.) can be supported later. RESERVED_12, RESERVED_13, RESERVED_14, RESERVED_15, RESERVED_16, RESERVED_17, RESERVED_18, RESERVED_19, RESERVED_20 } ``` For an example of how this is used, consider the `GetNeighbors` query. If we set the `edgeType` field of the `GfsNeighborsRequest`, the response will contain all the users that the specified user follows. If, on the other hand, we set `edgeType` to be `FollowedBy` it will return all the users who are followed by the specified user. ## FeatureType `FeatureType` is a thrift struct which is used in queries which require two edge types. ```thrift struct FeatureType { 1: required EdgeType leftEdgeType // edge type from source user 2: required EdgeType rightEdgeType // edge type from candidate user }(persisted="true") ``` ## UserWithScore The candidate generation queries return lists of candidates together with a computed score for the relevant feature. `UserWithScore` is a thrift struct which bundles together a candidate's ID with the score. ```thrift struct UserWithScore { 1: required i64 userId 2: required double score } ``` ================================================ FILE: graph-feature-service/doc/getintersection.md ================================================ # GetIntersection ## Request and response syntax A `GetIntersection` call takes as input a `GfsIntersectionRequest` thrift struct. ```thrift struct GfsIntersectionRequest { 1: required i64 userId 2: required list candidateUserIds 3: required list featureTypes } ``` The response is returned in a `GfsIntersectionResponse` thrift struct. ```thrift struct GfsIntersectionResponse { 1: required i64 userId 2: required list results } struct GfsIntersectionResult { 1: required i64 candidateUserId 2: required list intersectionValues } struct IntersectionValue { 1: required FeatureType featureType 2: optional i32 count 3: optional list intersectionIds 4: optional i32 leftNodeDegree 5: optional i32 rightNodeDegree }(persisted="true") ``` ## Behavior The `GfsIntersectionResponse` contains in its `results` field a `GfsIntersectionResult` for every candidate in `candidateIds` which contains an `IntersectionValue` for every `FeatureType` in the request's `featureTypes` field. The `IntersectionValue` contains the size of the intersection between the `leftEdgeType` edges from `userId` and the `rightEdgeType` edges from `candidateId` in the `count` field, as well as their respective degrees in the graphs in `leftNodeDegree` and `rightNodeDegree` respectively. **Note:** the `intersectionIds` field currently only contains `Nil`. ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common/BUILD.bazel ================================================ scala_library( platform = "java8", tags = [ "bazel-compatible", "bazel-only", ], dependencies = ["src/scala/com/twitter/storehaus_internal/util"], ) ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common/Configs.scala ================================================ package com.twitter.graph_feature_service.common import com.twitter.conversions.DurationOps._ import com.twitter.util.Duration import com.twitter.util.Time import java.nio.ByteBuffer import scala.util.hashing.MurmurHash3 object Configs { // NOTE: notify #recos-platform slack room, if you want to change this. // This SHOULD be updated together with NUM_SHARDS in worker.aurora final val NumGraphShards: Int = 40 final val TopKRealGraph: Int = 512 final val BaseHdfsPath: String = "/user/cassowary/processed/gfs/constant_db/" // whether or not to write in_value and out_value graphs. Used in the scalding job. final val EnableValueGraphs: Boolean = true // whether or not to write in_key and out_key graphs. Used in the scalding job. final val EnableKeyGraphs: Boolean = false final val FollowOutValPath: String = "follow_out_val/" final val FollowOutKeyPath: String = "follow_out_key/" final val FollowInValPath: String = "follow_in_val/" final val FollowInKeyPath: String = "follow_in_key/" final val MutualFollowValPath: String = "mutual_follow_val/" final val MutualFollowKeyPath: String = "mutual_follow_key/" final val FavoriteOutValPath: String = "favorite_out_val/" final val FavoriteInValPath: String = "favorite_in_val/" final val FavoriteOutKeyPath: String = "favorite_out_key/" final val FavoriteInKeyPath: String = "favorite_in_key/" final val RetweetOutValPath: String = "retweet_out_val/" final val RetweetInValPath: String = "retweet_in_val/" final val RetweetOutKeyPath: String = "retweet_out_key/" final val RetweetInKeyPath: String = "retweet_in_key/" final val MentionOutValPath: String = "mention_out_val/" final val MentionInValPath: String = "mention_in_val/" final val MentionOutKeyPath: String = "mention_out_key/" final val MentionInKeyPath: String = "mention_in_key/" final val MemCacheTTL: Duration = 8.hours final val RandomSeed: Int = 39582942 def getTimedHdfsShardPath(shardId: Int, path: String, time: Time): String = { val timeStr = time.format("yyyy/MM/dd") s"$path/$timeStr/shard_$shardId" } def getHdfsPath(path: String, overrideBaseHdfsPath: Option[String] = None): String = { val basePath = overrideBaseHdfsPath.getOrElse(BaseHdfsPath) s"$basePath$path" } private def hash(kArr: Array[Byte], seed: Int): Int = { MurmurHash3.bytesHash(kArr, seed) & 0x7fffffff // keep positive } private def hashLong(l: Long, seed: Int): Int = { hash(ByteBuffer.allocate(8).putLong(l).array(), seed) } def shardForUser(userId: Long): Int = { hashLong(userId, RandomSeed) % NumGraphShards } } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/BUILD.bazel ================================================ scala_library( sources = ["**/*.scala"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/twitter/storehaus:core", "3rdparty/jvm/javax/inject:javax.inject", "3rdparty/jvm/net/codingwell:scala-guice", "3rdparty/jvm/org/lz4:lz4-java", "3rdparty/jvm/org/slf4j:slf4j-api", "discovery-common/src/main/scala/com/twitter/discovery/common/stats", "finagle/finagle-http/src/main/scala", "finatra-internal/decider/src/main/scala", "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-app/src/main/scala", "finatra/inject/inject-core/src/main/scala", "finatra/inject/inject-server/src/main/scala", "finatra/inject/inject-thrift-client/src/main/scala", "finatra/inject/inject-utils/src/main/scala", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift:controller", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/filters", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/routing", "graph-feature-service/src/main/resources", "graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common", "graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util", "graph-feature-service/src/main/thrift/com/twitter/graph_feature_service:graph_feature_service_thrift-scala", "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", "servo/request/src/main/scala", "src/scala/com/twitter/storehaus_internal/memcache", "util/util-app/src/main/scala", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/Main.scala ================================================ package com.twitter.graph_feature_service.server import com.google.inject.Module import com.twitter.finatra.decider.modules.DeciderModule import com.twitter.finatra.mtls.thriftmux.Mtls import com.twitter.finatra.thrift.ThriftServer import com.twitter.finatra.thrift.filters.{ AccessLoggingFilter, LoggingMDCFilter, StatsFilter, ThriftMDCFilter, TraceIdMDCFilter } import com.twitter.finatra.mtls.thriftmux.modules.MtlsThriftWebFormsModule import com.twitter.finatra.thrift.routing.ThriftRouter import com.twitter.graph_feature_service.server.controllers.ServerController import com.twitter.graph_feature_service.server.handlers.ServerWarmupHandler import com.twitter.graph_feature_service.server.modules.{ GetIntersectionStoreModule, GraphFeatureServiceWorkerClientsModule, ServerFlagsModule } import com.twitter.graph_feature_service.thriftscala import com.twitter.inject.thrift.modules.ThriftClientIdModule object Main extends ServerMain class ServerMain extends ThriftServer with Mtls { override val name = "graph_feature_service-server" override val modules: Seq[Module] = { Seq( ServerFlagsModule, DeciderModule, ThriftClientIdModule, GraphFeatureServiceWorkerClientsModule, GetIntersectionStoreModule, new MtlsThriftWebFormsModule[thriftscala.Server.MethodPerEndpoint](this) ) } override def configureThrift(router: ThriftRouter): Unit = { router .filter[LoggingMDCFilter] .filter[TraceIdMDCFilter] .filter[ThriftMDCFilter] .filter[AccessLoggingFilter] .filter[StatsFilter] .add[ServerController] } override protected def warmup(): Unit = { handle[ServerWarmupHandler]() } } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/controllers/ServerController.scala ================================================ package com.twitter.graph_feature_service.server.controllers import com.twitter.discovery.common.stats.DiscoveryStatsFilter import com.twitter.finagle.Service import com.twitter.finagle.stats.StatsReceiver import com.twitter.finatra.thrift.Controller import com.twitter.graph_feature_service.server.handlers.ServerGetIntersectionHandler.GetIntersectionRequest import com.twitter.graph_feature_service.server.handlers.ServerGetIntersectionHandler import com.twitter.graph_feature_service.thriftscala import com.twitter.graph_feature_service.thriftscala.Server.GetIntersection import com.twitter.graph_feature_service.thriftscala.Server.GetPresetIntersection import com.twitter.graph_feature_service.thriftscala._ import javax.inject.Inject import javax.inject.Singleton @Singleton class ServerController @Inject() ( serverGetIntersectionHandler: ServerGetIntersectionHandler )( implicit statsReceiver: StatsReceiver) extends Controller(thriftscala.Server) { private val getIntersectionService: Service[GetIntersectionRequest, GfsIntersectionResponse] = new DiscoveryStatsFilter(statsReceiver.scope("srv").scope("get_intersection")) .andThen(Service.mk(serverGetIntersectionHandler)) val getIntersection: Service[GetIntersection.Args, GfsIntersectionResponse] = { args => // TODO: Disable updateCache after HTL switch to use PresetIntersection endpoint. getIntersectionService( GetIntersectionRequest.fromGfsIntersectionRequest(args.request, cacheable = true)) } handle(GetIntersection) { getIntersection } def getPresetIntersection: Service[ GetPresetIntersection.Args, GfsIntersectionResponse ] = { args => // TODO: Refactor after HTL switch to PresetIntersection val cacheable = args.request.presetFeatureTypes == PresetFeatureTypes.HtlTwoHop getIntersectionService( GetIntersectionRequest.fromGfsPresetIntersectionRequest(args.request, cacheable)) } handle(GetPresetIntersection) { getPresetIntersection } } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/handlers/ServerGetIntersectionHandler.scala ================================================ package com.twitter.graph_feature_service.server.handlers import com.twitter.finagle.stats.Stat import com.twitter.finagle.stats.StatsReceiver import com.twitter.graph_feature_service.server.handlers.ServerGetIntersectionHandler.GetIntersectionRequest import com.twitter.graph_feature_service.server.stores.FeatureTypesEncoder import com.twitter.graph_feature_service.server.stores.GetIntersectionStore.GetIntersectionQuery import com.twitter.graph_feature_service.thriftscala.PresetFeatureTypes import com.twitter.graph_feature_service.thriftscala._ import com.twitter.graph_feature_service.util.FeatureTypesCalculator import com.twitter.servo.request.RequestHandler import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import com.twitter.util.Memoize import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class ServerGetIntersectionHandler @Inject() ( @Named("ReadThroughGetIntersectionStore") readThroughStore: ReadableStore[GetIntersectionQuery, CachedIntersectionResult], @Named("BypassCacheGetIntersectionStore") readOnlyStore: ReadableStore[GetIntersectionQuery, CachedIntersectionResult] )( implicit statsReceiver: StatsReceiver) extends RequestHandler[GetIntersectionRequest, GfsIntersectionResponse] { import ServerGetIntersectionHandler._ // TODO: Track all the stats based on PresetFeatureType and update the dashboard private val stats: StatsReceiver = statsReceiver.scope("srv").scope("get_intersection") private val numCandidatesCount = stats.counter("total_num_candidates") private val numCandidatesStat = stats.stat("num_candidates") private val numFeaturesStat = stats.stat("num_features") private val userEmptyCount = stats.counter("user_empty_count") private val candidateEmptyRateStat = stats.stat("candidate_empty_rate") private val candidateNumEmptyStat = stats.stat("candidate_num_empty") private val missedRateStat = stats.stat("miss_rate") private val numMissedStat = stats.stat("num_missed") // Assume the order from HTL doesn't change. Only log the HTL query now. private val featureStatMap = FeatureTypesCalculator.presetFeatureTypes.map { feature => val featureString = s"${feature.leftEdgeType.name}_${feature.rightEdgeType.name}" feature -> Array( stats.counter(s"feature_type_${featureString}_total"), stats.counter(s"feature_type_${featureString}_count_zero"), stats.counter(s"feature_type_${featureString}_left_zero"), stats.counter(s"feature_type_${featureString}_right_zero") ) }.toMap private val sourceCandidateNumStats = Memoize[PresetFeatureTypes, Stat] { presetFeature => stats.stat(s"source_candidate_num_${presetFeature.name}") } override def apply(request: GetIntersectionRequest): Future[GfsIntersectionResponse] = { val featureTypes = request.calculatedFeatureTypes val numCandidates = request.candidateUserIds.length val numFeatures = featureTypes.length numCandidatesCount.incr(numCandidates) numCandidatesStat.add(numCandidates) numFeaturesStat.add(numFeatures) sourceCandidateNumStats(request.presetFeatureTypes).add(numCandidates) // Note: do not change the orders of features and candidates. val candidateIds = request.candidateUserIds if (featureTypes.isEmpty || candidateIds.isEmpty) { Future.value(DefaultGfsIntersectionResponse) } else { Future .collect { val getIntersectionStore = if (request.cacheable) readThroughStore else readOnlyStore getIntersectionStore.multiGet(GetIntersectionQuery.buildQueries(request)) }.map { responses => val results = responses.collect { case (query, Some(result)) => query.candidateId -> GfsIntersectionResult( query.candidateId, query.calculatedFeatureTypes.zip(result.values).map { case (featureType, value) => IntersectionValue( featureType, Some(value.count), if (value.intersectionIds.isEmpty) None else Some(value.intersectionIds), Some(value.leftNodeDegree), Some(value.rightNodeDegree) ) } ) } // Keep the response order same as input val processedResults = candidateIds.map { candidateId => results.getOrElse(candidateId, GfsIntersectionResult(candidateId, List.empty)) } val candidateEmptyNum = processedResults.count( _.intersectionValues.exists(value => isZero(value.rightNodeDegree))) val numMissed = processedResults.count(_.intersectionValues.size != numFeatures) if (processedResults.exists( _.intersectionValues.forall(value => isZero(value.leftNodeDegree)))) { userEmptyCount.incr() } candidateNumEmptyStat.add(candidateEmptyNum) candidateEmptyRateStat.add(candidateEmptyNum.toFloat / numCandidates) numMissedStat.add(numMissed) missedRateStat.add(numMissed.toFloat / numCandidates) processedResults.foreach { result => result.intersectionValues.zip(featureTypes).foreach { case (value, featureType) => featureStatMap.get(featureType).foreach { statsArray => statsArray(TotalIndex).incr() if (isZero(value.count)) { statsArray(CountIndex).incr() } if (isZero(value.leftNodeDegree)) { statsArray(LeftIndex).incr() } if (isZero(value.rightNodeDegree)) { statsArray(RightIndex).incr() } } } } GfsIntersectionResponse(processedResults) } } } } private[graph_feature_service] object ServerGetIntersectionHandler { case class GetIntersectionRequest( userId: Long, candidateUserIds: Seq[Long], featureTypes: Seq[FeatureType], presetFeatureTypes: PresetFeatureTypes, intersectionIdLimit: Option[Int], cacheable: Boolean) { lazy val calculatedFeatureTypes: Seq[FeatureType] = FeatureTypesCalculator.getFeatureTypes(presetFeatureTypes, featureTypes) lazy val calculatedFeatureTypesString: String = FeatureTypesEncoder(calculatedFeatureTypes) } object GetIntersectionRequest { def fromGfsIntersectionRequest( request: GfsIntersectionRequest, cacheable: Boolean ): GetIntersectionRequest = { GetIntersectionRequest( request.userId, request.candidateUserIds, request.featureTypes, PresetFeatureTypes.Empty, request.intersectionIdLimit, cacheable) } def fromGfsPresetIntersectionRequest( request: GfsPresetIntersectionRequest, cacheable: Boolean ): GetIntersectionRequest = { GetIntersectionRequest( request.userId, request.candidateUserIds, List.empty, request.presetFeatureTypes, request.intersectionIdLimit, cacheable) } } private val DefaultGfsIntersectionResponse = GfsIntersectionResponse() private val TotalIndex = 0 private val CountIndex = 1 private val LeftIndex = 2 private val RightIndex = 3 def isZero(opt: Option[Int]): Boolean = { !opt.exists(_ != 0) } } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/handlers/ServerWarmupHandler.scala ================================================ package com.twitter.graph_feature_service.server.handlers import com.twitter.finatra.thrift.routing.ThriftWarmup import com.twitter.graph_feature_service.thriftscala.EdgeType.FavoritedBy import com.twitter.graph_feature_service.thriftscala.EdgeType.FollowedBy import com.twitter.graph_feature_service.thriftscala.EdgeType.Following import com.twitter.graph_feature_service.thriftscala.Server.GetIntersection import com.twitter.graph_feature_service.thriftscala.FeatureType import com.twitter.graph_feature_service.thriftscala.GfsIntersectionRequest import com.twitter.inject.utils.Handler import com.twitter.scrooge.Request import com.twitter.util.logging.Logger import javax.inject.Inject import javax.inject.Singleton import scala.util.Random @Singleton class ServerWarmupHandler @Inject() (warmup: ThriftWarmup) extends Handler { val logger: Logger = Logger("WarmupHandler") // TODO: Add the testing accounts to warm-up the service. private val testingAccounts: Array[Long] = Seq.empty.toArray private def getRandomRequest: GfsIntersectionRequest = { GfsIntersectionRequest( testingAccounts(Random.nextInt(testingAccounts.length)), testingAccounts, Seq(FeatureType(Following, FollowedBy), FeatureType(Following, FavoritedBy)) ) } override def handle(): Unit = { warmup.sendRequest( GetIntersection, Request( GetIntersection.Args( getRandomRequest )), 10 )() logger.info("Warmup Done!") } } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/GetIntersectionStoreModule.scala ================================================ package com.twitter.graph_feature_service.server.modules import com.google.inject.Provides import com.twitter.bijection.scrooge.CompactScalaCodec import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.graph_feature_service.common.Configs._ import com.twitter.graph_feature_service.server.stores.GetIntersectionStore import com.twitter.graph_feature_service.server.stores.GetIntersectionStore.GetIntersectionQuery import com.twitter.graph_feature_service.thriftscala.CachedIntersectionResult import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.inject.TwitterModule import com.twitter.inject.annotations.Flag import com.twitter.storehaus.ReadableStore import com.twitter.storehaus_internal.memcache.MemcacheStore import com.twitter.storehaus_internal.util.{ClientName, ZkEndPoint} import com.twitter.util.Duration import javax.inject.{Named, Singleton} /** * Initialize the MemCache based GetIntersectionStore. * The Key of MemCache is UserId~CandidateId~FeatureTypes~IntersectionIdLimit. */ object GetIntersectionStoreModule extends TwitterModule { private[this] val requestTimeout: Duration = 25.millis private[this] val retries: Int = 0 @Provides @Named("ReadThroughGetIntersectionStore") @Singleton def provideReadThroughGetIntersectionStore( graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients, serviceIdentifier: ServiceIdentifier, @Flag(ServerFlagNames.MemCacheClientName) memCacheName: String, @Flag(ServerFlagNames.MemCachePath) memCachePath: String )( implicit statsReceiver: StatsReceiver ): ReadableStore[GetIntersectionQuery, CachedIntersectionResult] = { buildMemcacheStore( graphFeatureServiceWorkerClients, memCacheName, memCachePath, serviceIdentifier) } @Provides @Named("BypassCacheGetIntersectionStore") @Singleton def provideReadOnlyGetIntersectionStore( graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients, )( implicit statsReceiver: StatsReceiver ): ReadableStore[GetIntersectionQuery, CachedIntersectionResult] = { // Bypass the Memcache. GetIntersectionStore(graphFeatureServiceWorkerClients, statsReceiver) } private[this] def buildMemcacheStore( graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients, memCacheName: String, memCachePath: String, serviceIdentifier: ServiceIdentifier, )( implicit statsReceiver: StatsReceiver ): ReadableStore[GetIntersectionQuery, CachedIntersectionResult] = { val backingStore = GetIntersectionStore(graphFeatureServiceWorkerClients, statsReceiver) val cacheClient = MemcacheStore.memcachedClient( name = ClientName(memCacheName), dest = ZkEndPoint(memCachePath), timeout = requestTimeout, retries = retries, serviceIdentifier = serviceIdentifier, statsReceiver = statsReceiver ) ObservedMemcachedReadableStore.fromCacheClient[GetIntersectionQuery, CachedIntersectionResult]( backingStore = backingStore, cacheClient = cacheClient, ttl = MemCacheTTL )( valueInjection = LZ4Injection.compose(CompactScalaCodec(CachedIntersectionResult)), statsReceiver = statsReceiver.scope("mem_cache"), keyToString = { key => s"L~${key.userId}~${key.candidateId}~${key.featureTypesString}~${key.intersectionIdLimit}" } ) } } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/GraphFeatureServiceWorkerClientsModule.scala ================================================ package com.twitter.graph_feature_service.server.modules import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.mtls.client.MtlsStackClient._ import com.twitter.finagle.ThriftMux import com.twitter.finagle.service.RetryBudget import com.twitter.graph_feature_service.thriftscala import com.twitter.inject.TwitterModule import com.twitter.inject.annotations.Flag import com.twitter.util.{Await, Duration} import javax.inject.Singleton case class GraphFeatureServiceWorkerClients( workers: Seq[thriftscala.Worker.MethodPerEndpoint]) object GraphFeatureServiceWorkerClientsModule extends TwitterModule { private[this] val closeableGracePeriod: Duration = 1.second private[this] val requestTimeout: Duration = 25.millis @Provides @Singleton def provideGraphFeatureServiceWorkerClient( @Flag(ServerFlagNames.NumWorkers) numWorkers: Int, @Flag(ServerFlagNames.ServiceRole) serviceRole: String, @Flag(ServerFlagNames.ServiceEnv) serviceEnv: String, serviceIdentifier: ServiceIdentifier ): GraphFeatureServiceWorkerClients = { val workers: Seq[thriftscala.Worker.MethodPerEndpoint] = (0 until numWorkers).map { id => val dest = s"/srv#/$serviceEnv/local/$serviceRole/graph_feature_service-worker-$id" val client = ThriftMux.client .withRequestTimeout(requestTimeout) .withRetryBudget(RetryBudget.Empty) .withMutualTls(serviceIdentifier) .build[thriftscala.Worker.MethodPerEndpoint](dest, s"worker-$id") onExit { val closeable = client.asClosable Await.result(closeable.close(closeableGracePeriod), closeableGracePeriod) } client } GraphFeatureServiceWorkerClients(workers) } } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/LZ4Injection.scala ================================================ package com.twitter.graph_feature_service.server.modules import com.twitter.bijection.Injection import scala.util.Try import net.jpountz.lz4.{LZ4CompressorWithLength, LZ4DecompressorWithLength, LZ4Factory} object LZ4Injection extends Injection[Array[Byte], Array[Byte]] { private val lz4Factory = LZ4Factory.fastestInstance() private val fastCompressor = new LZ4CompressorWithLength(lz4Factory.fastCompressor()) private val decompressor = new LZ4DecompressorWithLength(lz4Factory.fastDecompressor()) override def apply(a: Array[Byte]): Array[Byte] = LZ4Injection.fastCompressor.compress(a) override def invert(b: Array[Byte]): Try[Array[Byte]] = Try { LZ4Injection.decompressor.decompress(b) } } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/ServerFlagModule.scala ================================================ package com.twitter.graph_feature_service.server.modules import com.twitter.inject.TwitterModule object ServerFlagNames { final val NumWorkers = "service.num_workers" final val ServiceRole = "service.role" final val ServiceEnv = "service.env" final val MemCacheClientName = "service.mem_cache_client_name" final val MemCachePath = "service.mem_cache_path" } /** * Initializes references to the flag values defined in the aurora.deploy file. * To check what the flag values are initialized in runtime, search FlagsModule in stdout */ object ServerFlagsModule extends TwitterModule { import ServerFlagNames._ flag[Int](NumWorkers, "Num of workers") flag[String](ServiceRole, "Service Role") flag[String](ServiceEnv, "Service Env") flag[String](MemCacheClientName, "MemCache Client Name") flag[String](MemCachePath, "MemCache Path") } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/stores/FeatureTypesEncoder.scala ================================================ package com.twitter.graph_feature_service.server.stores import com.twitter.graph_feature_service.common.Configs.RandomSeed import com.twitter.graph_feature_service.thriftscala.FeatureType import scala.util.hashing.MurmurHash3 object FeatureTypesEncoder { def apply(featureTypes: Seq[FeatureType]): String = { val byteArray = featureTypes.flatMap { featureType => Array(featureType.leftEdgeType.getValue.toByte, featureType.rightEdgeType.getValue.toByte) }.toArray (MurmurHash3.bytesHash(byteArray, RandomSeed) & 0x7fffffff).toString // keep positive } } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/stores/GetIntersectionStore.scala ================================================ package com.twitter.graph_feature_service.server.stores import com.twitter.finagle.RequestTimeoutException import com.twitter.finagle.stats.{Stat, StatsReceiver} import com.twitter.graph_feature_service.server.handlers.ServerGetIntersectionHandler.GetIntersectionRequest import com.twitter.graph_feature_service.server.modules.GraphFeatureServiceWorkerClients import com.twitter.graph_feature_service.server.stores.GetIntersectionStore.GetIntersectionQuery import com.twitter.graph_feature_service.thriftscala._ import com.twitter.inject.Logging import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import javax.inject.Singleton import scala.collection.mutable.ArrayBuffer @Singleton case class GetIntersectionStore( graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients, statsReceiver: StatsReceiver) extends ReadableStore[GetIntersectionQuery, CachedIntersectionResult] with Logging { import GetIntersectionStore._ private val stats = statsReceiver.scope("get_intersection_store") private val requestCount = stats.counter(name = "request_count") private val aggregatorLatency = stats.stat("aggregator_latency") private val timeOutCounter = stats.counter("worker_timeouts") private val unknownErrorCounter = stats.counter("unknown_errors") override def multiGet[K1 <: GetIntersectionQuery]( ks: Set[K1] ): Map[K1, Future[Option[CachedIntersectionResult]]] = { if (ks.isEmpty) { Map.empty } else { requestCount.incr() val head = ks.head // We assume all the GetIntersectionQuery use the same userId and featureTypes val userId = head.userId val featureTypes = head.featureTypes val presetFeatureTypes = head.presetFeatureTypes val calculatedFeatureTypes = head.calculatedFeatureTypes val intersectionIdLimit = head.intersectionIdLimit val request = WorkerIntersectionRequest( userId, ks.map(_.candidateId).toArray, featureTypes, presetFeatureTypes, intersectionIdLimit ) val resultFuture = Future .collect( graphFeatureServiceWorkerClients.workers.map { worker => worker .getIntersection(request) .rescue { case _: RequestTimeoutException => timeOutCounter.incr() Future.value(DefaultWorkerIntersectionResponse) case e => unknownErrorCounter.incr() logger.error("Failure to load result.", e) Future.value(DefaultWorkerIntersectionResponse) } } ).map { responses => Stat.time(aggregatorLatency) { gfsIntersectionResponseAggregator( responses, calculatedFeatureTypes, request.candidateUserIds, intersectionIdLimit ) } } ks.map { query => query -> resultFuture.map(_.get(query.candidateId)) }.toMap } } /** * Function to merge GfsIntersectionResponse from workers into one result. */ private def gfsIntersectionResponseAggregator( responseList: Seq[WorkerIntersectionResponse], features: Seq[FeatureType], candidates: Seq[Long], intersectionIdLimit: Int ): Map[Long, CachedIntersectionResult] = { // Map of (candidate -> features -> type -> value) val cube = Array.fill[Int](candidates.length, features.length, 3)(0) // Map of (candidate -> features -> intersectionIds) val ids = Array.fill[Option[ArrayBuffer[Long]]](candidates.length, features.length)(None) val notZero = intersectionIdLimit != 0 for { response <- responseList (features, candidateIndex) <- response.results.zipWithIndex (workerValue, featureIndex) <- features.zipWithIndex } { cube(candidateIndex)(featureIndex)(CountIndex) += workerValue.count cube(candidateIndex)(featureIndex)(LeftDegreeIndex) += workerValue.leftNodeDegree cube(candidateIndex)(featureIndex)(RightDegreeIndex) += workerValue.rightNodeDegree if (notZero && workerValue.intersectionIds.nonEmpty) { val arrayBuffer = ids(candidateIndex)(featureIndex) match { case Some(buffer) => buffer case None => val buffer = ArrayBuffer[Long]() ids(candidateIndex)(featureIndex) = Some(buffer) buffer } val intersectionIds = workerValue.intersectionIds // Scan the intersectionId based on the Shard. The response order is consistent. if (arrayBuffer.size < intersectionIdLimit) { if (intersectionIds.size > intersectionIdLimit - arrayBuffer.size) { arrayBuffer ++= intersectionIds.slice(0, intersectionIdLimit - arrayBuffer.size) } else { arrayBuffer ++= intersectionIds } } } } candidates.zipWithIndex.map { case (candidate, candidateIndex) => candidate -> CachedIntersectionResult(features.indices.map { featureIndex => WorkerIntersectionValue( cube(candidateIndex)(featureIndex)(CountIndex), cube(candidateIndex)(featureIndex)(LeftDegreeIndex), cube(candidateIndex)(featureIndex)(RightDegreeIndex), ids(candidateIndex)(featureIndex).getOrElse(Nil) ) }) }.toMap } } object GetIntersectionStore { private[graph_feature_service] case class GetIntersectionQuery( userId: Long, candidateId: Long, featureTypes: Seq[FeatureType], presetFeatureTypes: PresetFeatureTypes, featureTypesString: String, calculatedFeatureTypes: Seq[FeatureType], intersectionIdLimit: Int) private[graph_feature_service] object GetIntersectionQuery { def buildQueries(request: GetIntersectionRequest): Set[GetIntersectionQuery] = { request.candidateUserIds.toSet.map { candidateId: Long => GetIntersectionQuery( request.userId, candidateId, request.featureTypes, request.presetFeatureTypes, request.calculatedFeatureTypesString, request.calculatedFeatureTypes, request.intersectionIdLimit.getOrElse(DefaultIntersectionIdLimit) ) } } } // Don't return the intersectionId for better performance private val DefaultIntersectionIdLimit = 0 private val DefaultWorkerIntersectionResponse = WorkerIntersectionResponse() private val CountIndex = 0 private val LeftDegreeIndex = 1 private val RightDegreeIndex = 2 } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/BUILD ================================================ scala_library( platform = "java8", tags = ["bazel-compatible"], dependencies = [ "graph-feature-service/src/main/thrift/com/twitter/graph_feature_service:graph_feature_service_thrift-scala", ], ) ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/FeatureTypesCalculator.scala ================================================ package com.twitter.graph_feature_service.util import com.twitter.graph_feature_service.thriftscala.EdgeType._ import com.twitter.graph_feature_service.thriftscala.{FeatureType, PresetFeatureTypes} object FeatureTypesCalculator { final val DefaultTwoHop = Seq( FeatureType(Following, FollowedBy), FeatureType(Following, FavoritedBy), FeatureType(Following, RetweetedBy), FeatureType(Following, MentionedBy), FeatureType(Following, MutualFollow), FeatureType(Favorite, FollowedBy), FeatureType(Favorite, FavoritedBy), FeatureType(Favorite, RetweetedBy), FeatureType(Favorite, MentionedBy), FeatureType(Favorite, MutualFollow), FeatureType(MutualFollow, FollowedBy), FeatureType(MutualFollow, FavoritedBy), FeatureType(MutualFollow, RetweetedBy), FeatureType(MutualFollow, MentionedBy), FeatureType(MutualFollow, MutualFollow) ) final val SocialProofTwoHop = Seq(FeatureType(Following, FollowedBy)) final val HtlTwoHop = DefaultTwoHop final val WtfTwoHop = SocialProofTwoHop final val SqTwoHop = DefaultTwoHop final val RuxTwoHop = DefaultTwoHop final val MRTwoHop = DefaultTwoHop final val UserTypeaheadTwoHop = SocialProofTwoHop final val presetFeatureTypes = (HtlTwoHop ++ WtfTwoHop ++ SqTwoHop ++ RuxTwoHop ++ MRTwoHop ++ UserTypeaheadTwoHop).toSet def getFeatureTypes( presetFeatureTypes: PresetFeatureTypes, featureTypes: Seq[FeatureType] ): Seq[FeatureType] = { presetFeatureTypes match { case PresetFeatureTypes.HtlTwoHop => HtlTwoHop case PresetFeatureTypes.WtfTwoHop => WtfTwoHop case PresetFeatureTypes.SqTwoHop => SqTwoHop case PresetFeatureTypes.RuxTwoHop => RuxTwoHop case PresetFeatureTypes.MrTwoHop => MRTwoHop case PresetFeatureTypes.UserTypeaheadTwoHop => UserTypeaheadTwoHop case _ => featureTypes } } } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util/IntersectionValueCalculator.scala ================================================ package com.twitter.graph_feature_service.util import com.twitter.graph_feature_service.thriftscala.{ FeatureType, IntersectionValue, WorkerIntersectionValue } import java.nio.ByteBuffer import scala.collection.mutable.ArrayBuffer /** * Functions for computing feature values based on the values returned by constantDB. */ object IntersectionValueCalculator { /** * Compute the size of the array in a ByteBuffer. * Note that this function assumes the ByteBuffer is encoded using Injections.seqLong2ByteBuffer */ def computeArraySize(x: ByteBuffer): Int = { x.remaining() >> 3 // divide 8 } /** * */ def apply(x: ByteBuffer, y: ByteBuffer, intersectionIdLimit: Int): WorkerIntersectionValue = { val xSize = computeArraySize(x) val ySize = computeArraySize(y) val largerArray = if (xSize > ySize) x else y val smallerArray = if (xSize > ySize) y else x if (intersectionIdLimit == 0) { val result = computeIntersectionUsingBinarySearchOnLargerByteBuffer(smallerArray, largerArray) WorkerIntersectionValue(result, xSize, ySize) } else { val (result, ids) = computeIntersectionWithIds(smallerArray, largerArray, intersectionIdLimit) WorkerIntersectionValue(result, xSize, ySize, ids) } } /** * Note that this function assumes the ByteBuffer is encoded using Injections.seqLong2ByteBuffer * */ def computeIntersectionUsingBinarySearchOnLargerByteBuffer( smallArray: ByteBuffer, largeArray: ByteBuffer ): Int = { var res: Int = 0 var i: Int = 0 while (i < smallArray.remaining()) { if (binarySearch(largeArray, smallArray.getLong(i)) >= 0) { res += 1 } i += 8 } res } def computeIntersectionWithIds( smallArray: ByteBuffer, largeArray: ByteBuffer, intersectionLimit: Int ): (Int, Seq[Long]) = { var res: Int = 0 var i: Int = 0 // Most of the intersectionLimit is smaller than default size: 16 val idBuffer = ArrayBuffer[Long]() while (i < smallArray.remaining()) { val value = smallArray.getLong(i) if (binarySearch(largeArray, value) >= 0) { res += 1 // Always get the smaller ids if (idBuffer.size < intersectionLimit) { idBuffer += value } } i += 8 } (res, idBuffer) } /** * Note that this function assumes the ByteBuffer is encoded using Injections.seqLong2ByteBuffer * */ private[util] def binarySearch(arr: ByteBuffer, value: Long): Int = { var start = 0 var end = arr.remaining() while (start <= end && start < arr.remaining()) { val mid = ((start + end) >> 1) & ~7 // take mid - mid % 8 if (arr.getLong(mid) == value) { return mid // return the index of the value } else if (arr.getLong(mid) < value) { start = mid + 8 } else { end = mid - 1 } } // if not existed, return -1 -1 } /** * TODO: for now it only computes intersection size. Will add more feature types (e.g., dot * product, maximum value). * * NOTE that this function assumes both x and y are SORTED arrays. * In graph feature service, the sorting is done in the offline Scalding job. * * @param x source user's array * @param y candidate user's array * @param featureType feature type * @return */ def apply(x: Array[Long], y: Array[Long], featureType: FeatureType): IntersectionValue = { val xSize = x.length val ySize = y.length val intersection = if (xSize.min(ySize) * math.log(xSize.max(ySize)) < (xSize + ySize).toDouble) { if (xSize < ySize) { computeIntersectionUsingBinarySearchOnLargerArray(x, y) } else { computeIntersectionUsingBinarySearchOnLargerArray(y, x) } } else { computeIntersectionUsingListMerging(x, y) } IntersectionValue( featureType, Some(intersection.toInt), None, // return None for now Some(xSize), Some(ySize) ) } /** * Function for computing the intersections of two SORTED arrays by list merging. * * @param x one array * @param y another array * @param ordering ordering function for comparing values of T * @tparam T type * @return The intersection size and the list of intersected elements */ private[util] def computeIntersectionUsingListMerging[T]( x: Array[T], y: Array[T] )( implicit ordering: Ordering[T] ): Int = { var res: Int = 0 var i: Int = 0 var j: Int = 0 while (i < x.length && j < y.length) { val comp = ordering.compare(x(i), y(j)) if (comp > 0) j += 1 else if (comp < 0) i += 1 else { res += 1 i += 1 j += 1 } } res } /** * Function for computing the intersections of two arrays by binary search on the larger array. * Note that the larger array MUST be SORTED. * * @param smallArray smaller array * @param largeArray larger array * @param ordering ordering function for comparing values of T * @tparam T type * * @return The intersection size and the list of intersected elements */ private[util] def computeIntersectionUsingBinarySearchOnLargerArray[T]( smallArray: Array[T], largeArray: Array[T] )( implicit ordering: Ordering[T] ): Int = { var res: Int = 0 var i: Int = 0 while (i < smallArray.length) { val currentValue: T = smallArray(i) if (binarySearch(largeArray, currentValue) >= 0) { res += 1 } i += 1 } res } /** * Function for doing the binary search * * @param arr array * @param value the target value for searching * @param ordering ordering function * @tparam T type * @return the index of element in the larger array. * If there is no such element in the array, return -1. */ private[util] def binarySearch[T]( arr: Array[T], value: T )( implicit ordering: Ordering[T] ): Int = { var start = 0 var end = arr.length - 1 while (start <= end) { val mid = (start + end) >> 1 val comp = ordering.compare(arr(mid), value) if (comp == 0) { return mid // return the index of the value } else if (comp < 0) { start = mid + 1 } else { end = mid - 1 } } // if not existed, return -1 -1 } } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/BUILD.bazel ================================================ scala_library( sources = ["**/*.scala"], platform = "java8", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/javax/inject:javax.inject", "3rdparty/jvm/net/codingwell:scala-guice", "discovery-common/src/main/scala/com/twitter/discovery/common/stats", "finatra-internal/decider/src/main/scala", "finatra-internal/gizmoduck/src/main/scala", "finatra-internal/mtls-thriftmux/src/main/scala", "finatra/inject/inject-app/src/main/scala", "finatra/inject/inject-core/src/main/scala", "finatra/inject/inject-server/src/main/scala", "finatra/inject/inject-thrift-client/src/main/scala", "finatra/inject/inject-utils/src/main/scala", "frigate/frigate-common:constdb_util", "graph-feature-service/src/main/resources", "graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common", "graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util", "graph-feature-service/src/main/thrift/com/twitter/graph_feature_service:graph_feature_service_thrift-scala", "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", "servo/request/src/main/scala", "twitter-server-internal/src/main/scala", "twitter-server/server/src/main/scala", "util/util-app/src/main/scala", "util/util-slf4j-api/src/main/scala", ], ) ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/Main.scala ================================================ package com.twitter.graph_feature_service.worker import com.google.inject.Module import com.twitter.finatra.decider.modules.DeciderModule import com.twitter.finatra.gizmoduck.modules.TimerModule import com.twitter.finatra.mtls.thriftmux.Mtls import com.twitter.finatra.thrift.ThriftServer import com.twitter.finatra.thrift.filters.{ LoggingMDCFilter, StatsFilter, ThriftMDCFilter, TraceIdMDCFilter } import com.twitter.finatra.mtls.thriftmux.modules.MtlsThriftWebFormsModule import com.twitter.finatra.thrift.routing.ThriftRouter import com.twitter.graph_feature_service.thriftscala import com.twitter.graph_feature_service.worker.controllers.WorkerController import com.twitter.graph_feature_service.worker.handlers.WorkerWarmupHandler import com.twitter.graph_feature_service.worker.modules.{ GraphContainerProviderModule, WorkerFlagModule } import com.twitter.graph_feature_service.worker.util.GraphContainer import com.twitter.inject.thrift.modules.ThriftClientIdModule import com.twitter.util.Await object Main extends WorkerMain class WorkerMain extends ThriftServer with Mtls { override val name = "graph_feature_service-worker" override val modules: Seq[Module] = { Seq( WorkerFlagModule, DeciderModule, TimerModule, ThriftClientIdModule, GraphContainerProviderModule, new MtlsThriftWebFormsModule[thriftscala.Worker.MethodPerEndpoint](this) ) } override def configureThrift(router: ThriftRouter): Unit = { router .filter[LoggingMDCFilter] .filter[TraceIdMDCFilter] .filter[ThriftMDCFilter] .filter[StatsFilter] .add[WorkerController] } override protected def warmup(): Unit = { val graphContainer = injector.instance[GraphContainer] Await.result(graphContainer.warmup) handle[WorkerWarmupHandler]() } } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/controllers/WorkerController.scala ================================================ package com.twitter.graph_feature_service.worker.controllers import com.twitter.discovery.common.stats.DiscoveryStatsFilter import com.twitter.finagle.Service import com.twitter.finagle.stats.StatsReceiver import com.twitter.finatra.thrift.Controller import com.twitter.graph_feature_service.thriftscala import com.twitter.graph_feature_service.thriftscala.Worker.GetIntersection import com.twitter.graph_feature_service.thriftscala._ import com.twitter.graph_feature_service.worker.handlers._ import javax.inject.Inject import javax.inject.Singleton @Singleton class WorkerController @Inject() ( workerGetIntersectionHandler: WorkerGetIntersectionHandler )( implicit statsReceiver: StatsReceiver) extends Controller(thriftscala.Worker) { // use DiscoveryStatsFilter to filter out exceptions out of our control private val getIntersectionService: Service[ WorkerIntersectionRequest, WorkerIntersectionResponse ] = new DiscoveryStatsFilter[WorkerIntersectionRequest, WorkerIntersectionResponse]( statsReceiver.scope("srv").scope("get_intersection") ).andThen(Service.mk(workerGetIntersectionHandler)) val getIntersection: Service[GetIntersection.Args, WorkerIntersectionResponse] = { args => getIntersectionService(args.request).onFailure { throwable => logger.error(s"Failure to get intersection for request $args.", throwable) } } handle(GetIntersection) { getIntersection } } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/handlers/WorkerGetIntersectionHandler.scala ================================================ package com.twitter.graph_feature_service.worker.handlers import com.twitter.finagle.stats.{Stat, StatsReceiver} import com.twitter.graph_feature_service.thriftscala.{ WorkerIntersectionRequest, WorkerIntersectionResponse, WorkerIntersectionValue } import com.twitter.graph_feature_service.util.{FeatureTypesCalculator, IntersectionValueCalculator} import com.twitter.graph_feature_service.util.IntersectionValueCalculator._ import com.twitter.graph_feature_service.worker.util.GraphContainer import com.twitter.servo.request.RequestHandler import com.twitter.util.Future import java.nio.ByteBuffer import javax.inject.{Inject, Singleton} @Singleton class WorkerGetIntersectionHandler @Inject() ( graphContainer: GraphContainer, statsReceiver: StatsReceiver) extends RequestHandler[WorkerIntersectionRequest, WorkerIntersectionResponse] { import WorkerGetIntersectionHandler._ private val stats: StatsReceiver = statsReceiver.scope("srv/get_intersection") private val numCandidatesCount = stats.counter("total_num_candidates") private val toPartialGraphQueryStat = stats.stat("to_partial_graph_query_latency") private val fromPartialGraphQueryStat = stats.stat("from_partial_graph_query_latency") private val intersectionCalculationStat = stats.stat("computation_latency") override def apply(request: WorkerIntersectionRequest): Future[WorkerIntersectionResponse] = { numCandidatesCount.incr(request.candidateUserIds.length) val userId = request.userId // NOTE: do not change the order of candidates val candidateIds = request.candidateUserIds // NOTE: do not change the order of features val featureTypes = FeatureTypesCalculator.getFeatureTypes(request.presetFeatureTypes, request.featureTypes) val leftEdges = featureTypes.map(_.leftEdgeType).distinct val rightEdges = featureTypes.map(_.rightEdgeType).distinct val rightEdgeMap = Stat.time(toPartialGraphQueryStat) { rightEdges.map { rightEdge => val map = graphContainer.toPartialMap.get(rightEdge) match { case Some(graph) => candidateIds.flatMap { candidateId => graph.apply(candidateId).map(candidateId -> _) }.toMap case None => Map.empty[Long, ByteBuffer] } rightEdge -> map }.toMap } val leftEdgeMap = Stat.time(fromPartialGraphQueryStat) { leftEdges.flatMap { leftEdge => graphContainer.toPartialMap.get(leftEdge).flatMap(_.apply(userId)).map(leftEdge -> _) }.toMap } val res = Stat.time(intersectionCalculationStat) { WorkerIntersectionResponse( // NOTE that candidate ordering is important candidateIds.map { candidateId => // NOTE that the featureTypes ordering is important featureTypes.map { featureType => val leftNeighborsOpt = leftEdgeMap.get(featureType.leftEdgeType) val rightNeighborsOpt = rightEdgeMap.get(featureType.rightEdgeType).flatMap(_.get(candidateId)) if (leftNeighborsOpt.isEmpty && rightNeighborsOpt.isEmpty) { EmptyWorkerIntersectionValue } else if (rightNeighborsOpt.isEmpty) { EmptyWorkerIntersectionValue.copy( leftNodeDegree = computeArraySize(leftNeighborsOpt.get) ) } else if (leftNeighborsOpt.isEmpty) { EmptyWorkerIntersectionValue.copy( rightNodeDegree = computeArraySize(rightNeighborsOpt.get) ) } else { IntersectionValueCalculator( leftNeighborsOpt.get, rightNeighborsOpt.get, request.intersectionIdLimit) } } } ) } Future.value(res) } } object WorkerGetIntersectionHandler { val EmptyWorkerIntersectionValue: WorkerIntersectionValue = WorkerIntersectionValue(0, 0, 0, Nil) } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/handlers/WorkerWarmupHandler.scala ================================================ package com.twitter.graph_feature_service.worker.handlers import com.twitter.finatra.thrift.routing.ThriftWarmup import com.twitter.inject.Logging import com.twitter.inject.utils.Handler import javax.inject.{Inject, Singleton} @Singleton class WorkerWarmupHandler @Inject() (warmup: ThriftWarmup) extends Handler with Logging { override def handle(): Unit = { info("Warmup Done!") } } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/modules/GraphContainerProviderModule.scala ================================================ package com.twitter.graph_feature_service.worker.modules import com.google.inject.Provides import com.twitter.concurrent.AsyncSemaphore import com.twitter.finagle.stats.StatsReceiver import com.twitter.graph_feature_service.common.Configs._ import com.twitter.graph_feature_service.worker.util import com.twitter.graph_feature_service.worker.util.AutoUpdatingGraph import com.twitter.graph_feature_service.worker.util.FollowedByPartialValueGraph import com.twitter.graph_feature_service.worker.util.FollowingPartialValueGraph import com.twitter.graph_feature_service.worker.util.GraphContainer import com.twitter.graph_feature_service.worker.util.GraphKey import com.twitter.graph_feature_service.worker.util.MutualFollowPartialValueGraph import com.twitter.inject.TwitterModule import com.twitter.inject.annotations.Flag import com.twitter.util.Timer import javax.inject.Singleton object GraphContainerProviderModule extends TwitterModule { @Provides @Singleton def provideAutoUpdatingGraphs( @Flag(WorkerFlagNames.HdfsCluster) hdfsCluster: String, @Flag(WorkerFlagNames.HdfsClusterUrl) hdfsClusterUrl: String, @Flag(WorkerFlagNames.ShardId) shardId: Int )( implicit statsReceiver: StatsReceiver, timer: Timer ): GraphContainer = { // NOTE that we do not load some the graphs for saving RAM at this moment. val enabledGraphPaths: Map[GraphKey, String] = Map( FollowingPartialValueGraph -> FollowOutValPath, FollowedByPartialValueGraph -> FollowInValPath ) // Only allow one graph to update at the same time. val sharedSemaphore = new AsyncSemaphore(1) val graphs: Map[GraphKey, AutoUpdatingGraph] = enabledGraphPaths.map { case (graphKey, path) => graphKey -> AutoUpdatingGraph( dataPath = getHdfsPath(path), hdfsCluster = hdfsCluster, hdfsClusterUrl = hdfsClusterUrl, shard = shardId, minimumSizeForCompleteGraph = 1e6.toLong, sharedSemaphore = Some(sharedSemaphore) )( statsReceiver .scope("graphs") .scope(graphKey.getClass.getSimpleName), timer ) } util.GraphContainer(graphs) } } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/modules/WorkerFlagModule.scala ================================================ package com.twitter.graph_feature_service.worker.modules import com.twitter.inject.TwitterModule object WorkerFlagNames { final val ServiceRole = "service.role" final val ServiceEnv = "service.env" final val ShardId = "service.shardId" final val NumShards = "service.numShards" final val HdfsCluster = "service.hdfsCluster" final val HdfsClusterUrl = "service.hdfsClusterUrl" } /** * Initializes references to the flag values defined in the aurora.deploy file. * To check what the flag values are initialized in runtime, search FlagsModule in stdout */ object WorkerFlagModule extends TwitterModule { import WorkerFlagNames._ flag[Int](ShardId, "Shard Id") flag[Int](NumShards, "Num of Graph Shards") flag[String](ServiceRole, "Service Role") flag[String](ServiceEnv, "Service Env") flag[String](HdfsCluster, "Hdfs cluster to download graph files from") flag[String](HdfsClusterUrl, "Hdfs cluster url to download graph files from") } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/AutoUpdatingGraph.scala ================================================ package com.twitter.graph_feature_service.worker.util import com.twitter.bijection.Injection import com.twitter.concurrent.AsyncSemaphore import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.StatsReceiver import com.twitter.frigate.common.constdb_util.{ AutoUpdatingReadOnlyGraph, ConstDBImporter, Injections } import com.twitter.graph_feature_service.common.Configs import com.twitter.util.{Duration, Future, Timer} import java.nio.ByteBuffer /** * @param dataPath the path to the data on HDFS * @param hdfsCluster cluster where we check for updates and download graph files from * @param hdfsClusterUrl url to HDFS cluster * @param shard The shard of the graph to download * @param minimumSizeForCompleteGraph minimumSize for complete graph - otherwise we don't load it * @param updateIntervalMin The interval after which the first update is tried and the interval between such updates * @param updateIntervalMax the maximum time before an update is triggered * @param deleteInterval The interval after which older data is deleted from disk * @param sharedSemaphore The semaphore controls the number of graph loads at same time on the instance. */ case class AutoUpdatingGraph( dataPath: String, hdfsCluster: String, hdfsClusterUrl: String, shard: Int, minimumSizeForCompleteGraph: Long, updateIntervalMin: Duration = 1.hour, updateIntervalMax: Duration = 12.hours, deleteInterval: Duration = 2.seconds, sharedSemaphore: Option[AsyncSemaphore] = None )( implicit statsReceiver: StatsReceiver, timer: Timer) extends AutoUpdatingReadOnlyGraph[Long, ByteBuffer]( hdfsCluster, hdfsClusterUrl, shard, minimumSizeForCompleteGraph, updateIntervalMin, updateIntervalMax, deleteInterval, sharedSemaphore ) with ConstDBImporter[Long, ByteBuffer] { override def numGraphShards: Int = Configs.NumGraphShards override def basePath: String = dataPath override val keyInj: Injection[Long, ByteBuffer] = Injections.long2Varint override val valueInj: Injection[ByteBuffer, ByteBuffer] = Injection.identity override def get(targetId: Long): Future[Option[ByteBuffer]] = super .get(targetId) .map { res => res.foreach(r => arraySizeStat.add(r.remaining())) res } private val arraySizeStat = stats.scope("get").stat("size") } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GfsQuery.scala ================================================ package com.twitter.graph_feature_service.worker.util import com.twitter.graph_feature_service.thriftscala.EdgeType sealed trait GfsQuery { def edgeType: EdgeType def userId: Long } /** * Search for edges for any users to users in local partition. */ case class ToPartialQuery(edgeType: EdgeType, userId: Long) extends GfsQuery ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphContainer.scala ================================================ package com.twitter.graph_feature_service.worker.util import com.twitter.graph_feature_service.thriftscala.EdgeType import com.twitter.util.Future case class GraphContainer( graphs: Map[GraphKey, AutoUpdatingGraph]) { final val toPartialMap: Map[EdgeType, AutoUpdatingGraph] = graphs.collect { case (partialValueGraph: PartialValueGraph, graph) => partialValueGraph.edgeType -> graph } // load all the graphs from constantDB format to memory def warmup: Future[Unit] = { Future.collect(graphs.mapValues(_.warmup())).unit } } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphKey.scala ================================================ package com.twitter.graph_feature_service.worker.util import com.twitter.graph_feature_service.thriftscala.EdgeType import com.twitter.graph_feature_service.thriftscala.EdgeType._ sealed trait GraphKey { def edgeType: EdgeType } sealed trait PartialValueGraph extends GraphKey /** * Follow Graphs */ object FollowingPartialValueGraph extends PartialValueGraph { override def edgeType: EdgeType = Following } object FollowedByPartialValueGraph extends PartialValueGraph { override def edgeType: EdgeType = FollowedBy } /** * Mutual Follow Graphs */ object MutualFollowPartialValueGraph extends PartialValueGraph { override def edgeType: EdgeType = MutualFollow } ================================================ FILE: graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker/util/GraphType.scala ================================================ package com.twitter.graph_feature_service.worker.util //These classes are to help the GraphContainer choose the right data structure to answer queries sealed trait GraphType object FollowGraph extends GraphType object FavoriteGraph extends GraphType object RetweetGraph extends GraphType object ReplyGraph extends GraphType object MentionGraph extends GraphType object MutualFollowGraph extends GraphType ================================================ FILE: graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/BUILD.bazel ================================================ scala_library( platform = "java8", tags = [ "bazel-compatible", "bazel-only", ], dependencies = [ "3rdparty/jvm/com/twitter/bijection:core", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/constdb_util", "graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common", "src/scala/com/twitter/interaction_graph/scio/agg_all:interaction_graph_history_aggregated_edge_snapshot-scala", "src/scala/com/twitter/interaction_graph/scio/ml/scores:real_graph_in_scores-scala", "src/scala/com/twitter/pluck/source/user_audits:user_audit_final-scala", "src/scala/com/twitter/scalding_internal/dalv2", "src/scala/com/twitter/scalding_internal/job", "src/scala/com/twitter/scalding_internal/job/analytics_batch", ], ) scalding_job( name = "graph_feature_service_adhoc_job", main = "com.twitter.graph_feature_service.scalding.GraphFeatureServiceAdhocApp", args = [ "--date 2022-10-24", ], config = [ ("hadoop.map.jvm.total-memory", "3072m"), ("hadoop.reduce.jvm.total-memory", "3072m"), ("hadoop.submitter.jvm.total-memory", "5120m"), ("submitter.tier", "preemptible"), ], contact = "recos-platform-alerts@twitter.com", hadoop_cluster = "atla-proc", hadoop_properties = [("mapreduce.job.hdfs-servers", "/atla/proc/user/cassowary")], platform = "java8", role = "cassowary", runtime_platform = "java8", tags = [ "bazel-compatible:migrated", "bazel-only", ], dependencies = [":scalding"], ) scalding_job( name = "graph_feature_service_daily_job", main = "com.twitter.graph_feature_service.scalding.GraphFeatureServiceScheduledApp", config = [ ("hadoop.map.jvm.total-memory", "3072m"), ("hadoop.reduce.jvm.total-memory", "3072m"), ("hadoop.submitter.jvm.total-memory", "5120m"), ("submitter.tier", "preemptible"), ], contact = "recos-platform-alerts@twitter.com", cron = "01,31 * * * *", hadoop_cluster = "atla-proc", hadoop_properties = [("mapreduce.job.hdfs-servers", "/atla/proc/user/cassowary")], platform = "java8", role = "cassowary", runtime_platform = "java8", tags = [ "bazel-compatible:migrated", "bazel-only", ], dependencies = [":scalding"], ) ================================================ FILE: graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/EdgeFeature.scala ================================================ package com.twitter.graph_feature_service.scalding case class EdgeFeature( realGraphScore: Float, followScore: Option[Float] = None, mutualFollowScore: Option[Float] = None, favoriteScore: Option[Float] = None, retweetScore: Option[Float] = None, mentionScore: Option[Float] = None) ================================================ FILE: graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceAppBase.scala ================================================ package com.twitter.graph_feature_service.scalding import com.twitter.scalding._ import com.twitter.scalding_internal.job.TwitterExecutionApp import com.twitter.scalding_internal.job.analytics_batch.{ AnalyticsBatchExecution, AnalyticsBatchExecutionArgs, BatchDescription, BatchFirstTime, BatchIncrement, TwitterScheduledExecutionApp } import java.util.TimeZone /** * Each job only needs to implement this runOnDateRange() function. It makes it easier for testing. */ trait GraphFeatureServiceBaseJob { implicit val timeZone: TimeZone = DateOps.UTC implicit val dateParser: DateParser = DateParser.default def runOnDateRange( enableValueGraphs: Option[Boolean] = None, enableKeyGraphs: Option[Boolean] = None )( implicit dateRange: DateRange, timeZone: TimeZone, uniqueID: UniqueID ): Execution[Unit] /** * Print customized counters in the log */ def printerCounters[T](execution: Execution[T]): Execution[Unit] = { execution.getCounters .flatMap { case (_, counters) => counters.toMap.toSeq .sortBy(e => (e._1.group, e._1.counter)) .foreach { case (statKey, value) => println(s"${statKey.group}\t${statKey.counter}\t$value") } Execution.unit } } } /** * Trait that wraps things about adhoc jobs. */ trait GraphFeatureServiceAdhocBaseApp extends TwitterExecutionApp with GraphFeatureServiceBaseJob { override def job: Execution[Unit] = Execution.withId { implicit uniqueId => Execution.getArgs.flatMap { args: Args => implicit val dateRange: DateRange = DateRange.parse(args.list("date"))(timeZone, dateParser) printerCounters(runOnDateRange()) } } } /** * Trait that wraps things about scheduled jobs. * * A new daily app only needs to declare the starting date. */ trait GraphFeatureServiceScheduledBaseApp extends TwitterScheduledExecutionApp with GraphFeatureServiceBaseJob { def firstTime: RichDate // for example: RichDate("2018-02-21") def batchIncrement: Duration = Days(1) override def scheduledJob: Execution[Unit] = Execution.withId { implicit uniqueId => val analyticsArgs = AnalyticsBatchExecutionArgs( batchDesc = BatchDescription(getClass.getName), firstTime = BatchFirstTime(firstTime), batchIncrement = BatchIncrement(batchIncrement) ) AnalyticsBatchExecution(analyticsArgs) { implicit dateRange => printerCounters(runOnDateRange()) } } } ================================================ FILE: graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceApps.scala ================================================ package com.twitter.graph_feature_service.scalding import com.twitter.scalding.DateRange import com.twitter.scalding.Execution import com.twitter.scalding.RichDate import com.twitter.scalding.UniqueID import java.util.Calendar import java.util.TimeZone import sun.util.calendar.BaseCalendar /** * To launch an adhoc run: * scalding remote run --target graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding:graph_feature_service_adhoc_job */ object GraphFeatureServiceAdhocApp extends GraphFeatureServiceMainJob with GraphFeatureServiceAdhocBaseApp {} /** * To schedule the job, upload the workflows config (only required for the first time and subsequent config changes): * scalding workflow upload --jobs graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding:graph_feature_service_daily_job --autoplay --build-cron-schedule "20 23 1 * *" * You can then build from the UI by clicking "Build" and pasting in your remote branch, or leave it empty if you're redeploying from master. * The workflows config above should automatically trigger once each month. */ object GraphFeatureServiceScheduledApp extends GraphFeatureServiceMainJob with GraphFeatureServiceScheduledBaseApp { override def firstTime: RichDate = RichDate("2018-05-18") override def runOnDateRange( enableValueGraphs: Option[Boolean], enableKeyGraphs: Option[Boolean] )( implicit dateRange: DateRange, timeZone: TimeZone, uniqueID: UniqueID ): Execution[Unit] = { // Only run the value Graphs on Tuesday, Thursday, Saturday val overrideEnableValueGraphs = { val dayOfWeek = dateRange.start.toCalendar.get(Calendar.DAY_OF_WEEK) dayOfWeek == BaseCalendar.TUESDAY | dayOfWeek == BaseCalendar.THURSDAY | dayOfWeek == BaseCalendar.SATURDAY } super.runOnDateRange( Some(true), Some(false) // disable key Graphs since we are not using them in production ) } } ================================================ FILE: graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/GraphFeatureServiceMainJob.scala ================================================ package com.twitter.graph_feature_service.scalding import com.twitter.bijection.Injection import com.twitter.frigate.common.constdb_util.Injections import com.twitter.frigate.common.constdb_util.ScaldingUtil import com.twitter.graph_feature_service.common.Configs import com.twitter.graph_feature_service.common.Configs._ import com.twitter.interaction_graph.scio.agg_all.InteractionGraphHistoryAggregatedEdgeSnapshotScalaDataset import com.twitter.interaction_graph.scio.ml.scores.RealGraphInScoresScalaDataset import com.twitter.interaction_graph.thriftscala.FeatureName import com.twitter.interaction_graph.thriftscala.{EdgeFeature => TEdgeFeature} import com.twitter.pluck.source.user_audits.UserAuditFinalScalaDataset import com.twitter.scalding.DateRange import com.twitter.scalding.Days import com.twitter.scalding.Execution import com.twitter.scalding.Stat import com.twitter.scalding.UniqueID import com.twitter.scalding.typed.TypedPipe import com.twitter.scalding_internal.dalv2.DAL import com.twitter.scalding_internal.dalv2.remote_access.AllowCrossClusterSameDC import com.twitter.scalding_internal.multiformat.format.keyval.KeyVal import com.twitter.util.Time import com.twitter.wtf.candidate.thriftscala.CandidateSeq import java.nio.ByteBuffer import java.util.TimeZone trait GraphFeatureServiceMainJob extends GraphFeatureServiceBaseJob { // keeping hdfsPath as a separate variable in order to override it in unit tests protected val hdfsPath: String = BaseHdfsPath protected def getShardIdForUser(userId: Long): Int = shardForUser(userId) protected implicit val keyInj: Injection[Long, ByteBuffer] = Injections.long2Varint protected implicit val valueInj: Injection[Long, ByteBuffer] = Injections.long2ByteBuffer protected val bufferSize: Int = 1 << 26 protected val maxNumKeys: Int = 1 << 24 protected val numReducers: Int = NumGraphShards protected val outputStreamBufferSize: Int = 1 << 26 protected final val shardingByKey = { (k: Long, _: Long) => getShardIdForUser(k) } protected final val shardingByValue = { (_: Long, v: Long) => getShardIdForUser(v) } private def writeGraphToDB( graph: TypedPipe[(Long, Long)], shardingFunction: (Long, Long) => Int, path: String )( implicit dateRange: DateRange ): Execution[TypedPipe[(Int, Unit)]] = { ScaldingUtil .writeConstDB[Long, Long]( graph.withDescription(s"sharding $path"), shardingFunction, shardId => getTimedHdfsShardPath( shardId, getHdfsPath(path, Some(hdfsPath)), Time.fromMilliseconds(dateRange.end.timestamp) ), Int.MaxValue, bufferSize, maxNumKeys, numReducers, outputStreamBufferSize )( keyInj, valueInj, Ordering[(Long, Long)] ) .forceToDiskExecution } def extractFeature( featureList: Seq[TEdgeFeature], featureName: FeatureName ): Option[Float] = { featureList .find(_.name == featureName) .map(_.tss.ewma.toFloat) .filter(_ > 0.0) } /** * Function to extract a subgraph (e.g., follow graph) from real graph and take top K by real graph * weight. * * @param input input real graph * @param edgeFilter filter function to only get the edges needed (e.g., only follow edges) * @param counter counter * @return a subgroup that contains topK, e.g., follow graph for each user. */ private def getSubGraph( input: TypedPipe[(Long, Long, EdgeFeature)], edgeFilter: EdgeFeature => Boolean, counter: Stat ): TypedPipe[(Long, Long)] = { input .filter(c => edgeFilter(c._3)) .map { case (srcId, destId, features) => (srcId, (destId, features.realGraphScore)) } .group // auto reducer estimation only allocates 15 reducers, so setting an explicit number here .withReducers(2000) .sortedReverseTake(TopKRealGraph)(Ordering.by(_._2)) .flatMap { case (srcId, topKNeighbors) => counter.inc() topKNeighbors.map { case (destId, _) => (srcId, destId) } } } def getMauIds()(implicit dateRange: DateRange, uniqueID: UniqueID): TypedPipe[Long] = { val numMAUs = Stat("NUM_MAUS") val uniqueMAUs = Stat("UNIQUE_MAUS") DAL .read(UserAuditFinalScalaDataset) .withRemoteReadPolicy(AllowCrossClusterSameDC) .toTypedPipe .collect { case user_audit if user_audit.isValid => numMAUs.inc() user_audit.userId } .distinct .map { u => uniqueMAUs.inc() u } } def getRealGraphWithMAUOnly( implicit dateRange: DateRange, timeZone: TimeZone, uniqueID: UniqueID ): TypedPipe[(Long, Long, EdgeFeature)] = { val numMAUs = Stat("NUM_MAUS") val uniqueMAUs = Stat("UNIQUE_MAUS") val monthlyActiveUsers = DAL .read(UserAuditFinalScalaDataset) .withRemoteReadPolicy(AllowCrossClusterSameDC) .toTypedPipe .collect { case user_audit if user_audit.isValid => numMAUs.inc() user_audit.userId } .distinct .map { u => uniqueMAUs.inc() u } .asKeys val realGraphAggregates = DAL .readMostRecentSnapshot( InteractionGraphHistoryAggregatedEdgeSnapshotScalaDataset, dateRange.embiggen(Days(5))) .withRemoteReadPolicy(AllowCrossClusterSameDC) .toTypedPipe .map { edge => val featureList = edge.features val edgeFeature = EdgeFeature( edge.weight.getOrElse(0.0).toFloat, extractFeature(featureList, FeatureName.NumMutualFollows), extractFeature(featureList, FeatureName.NumFavorites), extractFeature(featureList, FeatureName.NumRetweets), extractFeature(featureList, FeatureName.NumMentions) ) (edge.sourceId, (edge.destinationId, edgeFeature)) } .join(monthlyActiveUsers) .map { case (srcId, ((destId, feature), _)) => (destId, (srcId, feature)) } .join(monthlyActiveUsers) .map { case (destId, ((srcId, feature), _)) => (srcId, destId, feature) } realGraphAggregates } def getTopKFollowGraph( implicit dateRange: DateRange, timeZone: TimeZone, uniqueID: UniqueID ): TypedPipe[(Long, Long)] = { val followGraphMauStat = Stat("NumFollowEdges_MAU") val mau: TypedPipe[Long] = getMauIds() DAL .readMostRecentSnapshot(RealGraphInScoresScalaDataset, dateRange.embiggen(Days(7))) .withRemoteReadPolicy(AllowCrossClusterSameDC) .toTypedPipe .groupBy(_.key) .join(mau.asKeys) .withDescription("filtering srcId by mau") .flatMap { case (_, (KeyVal(srcId, CandidateSeq(candidates)), _)) => followGraphMauStat.inc() val topK = candidates.sortBy(-_.score).take(TopKRealGraph) topK.map { c => (srcId, c.userId) } } } override def runOnDateRange( enableValueGraphs: Option[Boolean], enableKeyGraphs: Option[Boolean] )( implicit dateRange: DateRange, timeZone: TimeZone, uniqueID: UniqueID ): Execution[Unit] = { val processValueGraphs = enableValueGraphs.getOrElse(Configs.EnableValueGraphs) val processKeyGraphs = enableKeyGraphs.getOrElse(Configs.EnableKeyGraphs) if (!processKeyGraphs && !processValueGraphs) { // Skip the batch job Execution.unit } else { // val favoriteGraphStat = Stat("NumFavoriteEdges") // val retweetGraphStat = Stat("NumRetweetEdges") // val mentionGraphStat = Stat("NumMentionEdges") // val realGraphAggregates = getRealGraphWithMAUOnly val followGraph = getTopKFollowGraph // val mutualFollowGraph = followGraph.asKeys.join(followGraph.swap.asKeys).keys // val favoriteGraph = // getSubGraph(realGraphAggregates, _.favoriteScore.isDefined, favoriteGraphStat) // val retweetGraph = // getSubGraph(realGraphAggregates, _.retweetScore.isDefined, retweetGraphStat) // val mentionGraph = // getSubGraph(realGraphAggregates, _.mentionScore.isDefined, mentionGraphStat) val writeValDataSetExecutions = if (processValueGraphs) { Seq( (followGraph, shardingByValue, FollowOutValPath), (followGraph.swap, shardingByValue, FollowInValPath) // (mutualFollowGraph, shardingByValue, MutualFollowValPath), // (favoriteGraph, shardingByValue, FavoriteOutValPath), // (favoriteGraph.swap, shardingByValue, FavoriteInValPath), // (retweetGraph, shardingByValue, RetweetOutValPath), // (retweetGraph.swap, shardingByValue, RetweetInValPath), // (mentionGraph, shardingByValue, MentionOutValPath), // (mentionGraph.swap, shardingByValue, MentionInValPath) ) } else { Seq.empty } val writeKeyDataSetExecutions = if (processKeyGraphs) { Seq( (followGraph, shardingByKey, FollowOutKeyPath), (followGraph.swap, shardingByKey, FollowInKeyPath) // (favoriteGraph, shardingByKey, FavoriteOutKeyPath), // (favoriteGraph.swap, shardingByKey, FavoriteInKeyPath), // (retweetGraph, shardingByKey, RetweetOutKeyPath), // (retweetGraph.swap, shardingByKey, RetweetInKeyPath), // (mentionGraph, shardingByKey, MentionOutKeyPath), // (mentionGraph.swap, shardingByKey, MentionInKeyPath), // (mutualFollowGraph, shardingByKey, MutualFollowKeyPath) ) } else { Seq.empty } Execution .sequence((writeValDataSetExecutions ++ writeKeyDataSetExecutions).map { case (graph, shardingMethod, path) => writeGraphToDB(graph, shardingMethod, path) }).unit } } } ================================================ FILE: graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc/BUILD.bazel ================================================ scala_library( platform = "java8", tags = ["bazel-only"], dependencies = [ "3rdparty/jvm/com/twitter/bijection:core", "3rdparty/jvm/com/twitter/bijection:scrooge", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/constdb_util", "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/ml/api:api-base", "src/scala/com/twitter/scalding_internal/job", "src/scala/com/twitter/scalding_internal/job/analytics_batch", "src/thrift/com/twitter/ml/api:data-java", ], ) hadoop_binary( name = "gfs_random_request-adhoc", main = "com.twitter.graph_feature_service.scalding.adhoc.RandomRequestGenerationApp", platform = "java8", runtime_platform = "java8", tags = [ "bazel-compatible", "bazel-compatible:migrated", "bazel-only", ], dependencies = [":adhoc"], ) ================================================ FILE: graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc/RandomRequestGenerationApp.scala ================================================ package com.twitter.graph_feature_service.scalding.adhoc import com.twitter.bijection.Injection import com.twitter.frigate.common.constdb_util.Injections import com.twitter.ml.api.Feature.Discrete import com.twitter.ml.api.{DailySuffixFeatureSource, DataSetPipe, RichDataRecord} import com.twitter.scalding._ import com.twitter.scalding_internal.job.TwitterExecutionApp import java.nio.ByteBuffer import java.util.TimeZone object RandomRequestGenerationJob { implicit val timeZone: TimeZone = DateOps.UTC implicit val dateParser: DateParser = DateParser.default val timelineRecapDataSetPath: String = "/atla/proc2/user/timelines/processed/suggests/recap/data_records" val USER_ID = new Discrete("meta.user_id") val AUTHOR_ID = new Discrete("meta.author_id") val timelineRecapOutPutPath: String = "/user/cassowary/gfs/adhoc/timeline_data" implicit val inj: Injection[Long, ByteBuffer] = Injections.long2Varint def run( dataSetPath: String, outPutPath: String, numOfPairsToTake: Int )( implicit dateRange: DateRange, uniqueID: UniqueID ): Execution[Unit] = { val NumUserAuthorPairs = Stat("NumUserAuthorPairs") val dataSet: DataSetPipe = DailySuffixFeatureSource(dataSetPath).read val userAuthorPairs: TypedPipe[(Long, Long)] = dataSet.records.map { record => val richRecord = new RichDataRecord(record, dataSet.featureContext) val userId = richRecord.getFeatureValue(USER_ID) val authorId = richRecord.getFeatureValue(AUTHOR_ID) NumUserAuthorPairs.inc() (userId, authorId) } userAuthorPairs .limit(numOfPairsToTake) .writeExecution( TypedTsv[(Long, Long)](outPutPath) ) } } /** * ./bazel bundle graph-feature-service/src/main/scalding/com/twitter/graph_feature_service/scalding/adhoc:all * * oscar hdfs --screen --user cassowary --tee gfs_log --bundle gfs_random_request-adhoc \ --tool com.twitter.graph_feature_service.scalding.adhoc.RandomRequestGenerationApp \ -- --date 2018-08-11 \ --input /atla/proc2/user/timelines/processed/suggests/recap/data_records \ --output /user/cassowary/gfs/adhoc/timeline_data */ object RandomRequestGenerationApp extends TwitterExecutionApp { import RandomRequestGenerationJob._ override def job: Execution[Unit] = Execution.withId { implicit uniqueId => Execution.getArgs.flatMap { args: Args => implicit val dateRange: DateRange = DateRange.parse(args.list("date"))(timeZone, dateParser) run( args.optional("input").getOrElse(timelineRecapDataSetPath), args.optional("output").getOrElse(timelineRecapOutPutPath), args.int("num_pairs", 3000) ) } } } ================================================ FILE: graph-feature-service/src/main/thrift/com/twitter/graph_feature_service/BUILD ================================================ create_thrift_libraries( base_name = "graph_feature_service_thrift", sources = ["*.thrift"], platform = "java8", tags = ["bazel-compatible"], generate_languages = [ "java", # ruby is added due to ruby dependees in timelines "ruby", "scala", "strato", ], provides_java_name = "graph_feature_service_thrift_java", provides_scala_name = "graph_feature_service_thrift_scala", ) ================================================ FILE: graph-feature-service/src/main/thrift/com/twitter/graph_feature_service/graph_feature_service.thrift ================================================ namespace java com.twitter.graph_feature_service.thriftjava #@namespace scala com.twitter.graph_feature_service.thriftscala #@namespace strato com.twitter.graph_feature_service.thriftscala // edge type to differentiate different types of graphs (we can also add a lot of other types of edges) enum EdgeType { FOLLOWING, FOLLOWED_BY, FAVORITE, FAVORITED_BY, RETWEET, RETWEETED_BY, REPLY, REPLYED_BY, MENTION, MENTIONED_BY, MUTUAL_FOLLOW, SIMILAR_TO, // more edge types (like block, report, etc.) can be supported later. RESERVED_12, RESERVED_13, RESERVED_14, RESERVED_15, RESERVED_16, RESERVED_17, RESERVED_18, RESERVED_19, RESERVED_20 } enum PresetFeatureTypes { EMPTY, HTL_TWO_HOP, WTF_TWO_HOP, SQ_TWO_HOP, RUX_TWO_HOP, MR_TWO_HOP, USER_TYPEAHEAD_TWO_HOP } struct UserWithCount { 1: required i64 userId(personalDataType = 'UserId') 2: required i32 count }(hasPersonalData = 'true') struct UserWithScore { 1: required i64 userId(personalDataType = 'UserId') 2: required double score }(hasPersonalData = 'true') // Feature Type // For example, to compute how many of source user's following's have favorited candidate user, // we need to compute the intersection between source user's FOLLOWING edges, and candidate user's // FAVORITED_BY edge. In this case, we should user FeatureType(FOLLOWING, FAVORITED_BY) struct FeatureType { 1: required EdgeType leftEdgeType // edge type from source user 2: required EdgeType rightEdgeType // edge type from candidate user }(persisted="true") struct IntersectionValue { 1: required FeatureType featureType 2: optional i32 count 3: optional list intersectionIds(personalDataType = 'UserId') 4: optional i32 leftNodeDegree 5: optional i32 rightNodeDegree }(persisted="true", hasPersonalData = 'true') struct GfsIntersectionResult { 1: required i64 candidateUserId(personalDataType = 'UserId') 2: required list intersectionValues }(hasPersonalData = 'true') struct GfsIntersectionRequest { 1: required i64 userId(personalDataType = 'UserId') 2: required list candidateUserIds(personalDataType = 'UserId') 3: required list featureTypes 4: optional i32 intersectionIdLimit } struct GfsPresetIntersectionRequest { 1: required i64 userId(personalDataType = 'UserId') 2: required list candidateUserIds(personalDataType = 'UserId') 3: required PresetFeatureTypes presetFeatureTypes 4: optional i32 intersectionIdLimit }(hasPersonalData = 'true') struct GfsIntersectionResponse { 1: required list results } service Server { GfsIntersectionResponse getIntersection(1: GfsIntersectionRequest request) GfsIntersectionResponse getPresetIntersection(1: GfsPresetIntersectionRequest request) } ################################################################################################### ## For internal usage only ################################################################################################### struct WorkerIntersectionRequest { 1: required i64 userId(personalDataType = 'UserId') 2: required list candidateUserIds(personalDataType = 'UserId') 3: required list featureTypes 4: required PresetFeatureTypes presetFeatureTypes 5: required i32 intersectionIdLimit }(hasPersonalData = 'true') struct WorkerIntersectionResponse { 1: required list> results } struct WorkerIntersectionValue { 1: i32 count 2: i32 leftNodeDegree 3: i32 rightNodeDegree 4: list intersectionIds(personalDataType = 'UserId') }(hasPersonalData = 'true') struct CachedIntersectionResult { 1: required list values } service Worker { WorkerIntersectionResponse getIntersection(1: WorkerIntersectionRequest request) } ================================================ FILE: home-mixer/BUILD.bazel ================================================ jvm_binary( name = "bin", basename = "home-mixer", main = "com.twitter.home_mixer.HomeMixerServerMain", runtime_platform = "java11", tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/ch/qos/logback:logback-classic", "finagle/finagle-zipkin-scribe/src/main/scala", "finatra/inject/inject-logback/src/main/scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer", "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", "twitter-server-internal/src/main/scala", "twitter-server/logback-classic/src/main/scala", ], ) # Aurora Workflows build phase convention requires a jvm_app named with home-mixer-app jvm_app( name = "home-mixer-app", archive = "zip", binary = ":bin", bundles = [ bundle( fileset = ["config/**/*"], owning_target = "home-mixer/config:files", ), ], tags = ["bazel-compatible"], ) ================================================ FILE: home-mixer/README.md ================================================ Home Mixer ========== Home Mixer is the main service used to construct and serve Twitter's Home Timelines. It currently powers: - For you - best Tweets from people you follow + recommended out-of-network content - Following - reverse chronological Tweets from people you follow - Lists - reverse chronological Tweets from List members Home Mixer is built on Product Mixer, our custom Scala framework that facilitates building feeds of content. ## Overview The For You recommendation algorithm in Home Mixer involves the following stages: - Candidate Generation - fetch Tweets from various Candidate Sources. For example: - Earlybird Search Index - User Tweet Entity Graph - Cr Mixer - Follow Recommendations Service - Feature Hydration - Fetch the ~6000 features needed for ranking - Scoring and Ranking using ML model - Filters and Heuristics. For example: - Author Diversity - Content Balance (In network vs Out of Network) - Feedback fatigue - Deduplication / previously seen Tweets removal - Visibility Filtering (blocked, muted authors/tweets, NSFW settings) - Mixing - integrate Tweets with non-Tweet content - Ads - Who-to-follow modules - Prompts - Product Features and Serving - Conversation Modules for replies - Social Context - Timeline Navigation - Edited Tweets - Feedback options - Pagination and cursoring - Observability and logging - Client instructions and content marshalling ## Pipeline Structure ### General Product Mixer services like Home Mixer are structured around Pipelines that split the execution into transparent and structured steps. Requests first go to Product Pipelines, which are used to select which Mixer Pipeline or Recommendation Pipeline to run for a given request. Each Mixer or Recommendation Pipeline may run multiple Candidate Pipelines to fetch candidates to include in the response. Mixer Pipelines combine the results of multiple heterogeneous Candidate Pipelines together (e.g. ads, tweets, users) while Recommendation Pipelines are used to score (via Scoring Pipelines) and rank the results of homogenous Candidate Pipelines so that the top ranked ones can be returned. These pipelines also marshall candidates into a domain object and then into a transport object to return to the caller. Candidate Pipelines fetch candidates from underlying Candidate Sources and perform some basic operations on the Candidates, such as filtering out unwanted candidates, applying decorations, and hydrating features. The sections below describe the high level pipeline structure (non-exhaustive) for the main Home Timeline tabs powered by Home Mixer. ### For You - ForYouProductPipelineConfig - ForYouScoredTweetsMixerPipelineConfig (main orchestration layer - mixes Tweets with ads and users) - ForYouScoredTweetsCandidatePipelineConfig (fetch Tweets) - ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer) - Fetch Tweet Candidates - ScoredTweetsInNetworkCandidatePipelineConfig - ScoredTweetsTweetMixerCandidatePipelineConfig - ScoredTweetsUtegCandidatePipelineConfig - ScoredTweetsFrsCandidatePipelineConfig - Feature Hydration and Scoring - ScoredTweetsScoringPipelineConfig - ForYouConversationServiceCandidatePipelineConfig (backup reverse chron pipeline in case Scored Tweets fails) - ForYouAdsCandidatePipelineConfig (fetch ads) - ForYouWhoToFollowCandidatePipelineConfig (fetch users to recommend) ### Following - FollowingProductPipelineConfig - FollowingMixerPipelineConfig - FollowingEarlybirdCandidatePipelineConfig (fetch tweets from Search Index) - ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules) - FollowingAdsCandidatePipelineConfig (fetch ads) - FollowingWhoToFollowCandidatePipelineConfig (fetch users to recommend) ### Lists - ListTweetsProductPipelineConfig - ListTweetsMixerPipelineConfig - ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service) - ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules) - ListTweetsAdsCandidatePipelineConfig (fetch ads) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "finagle/finagle-netty4/src/main/scala", "finatra-internal/mtls-http/src/main/scala", "home-mixer/server/src/main/resources", "home-mixer/server/src/main/scala/com/twitter/home_mixer/controller", "home-mixer/server/src/main/scala/com/twitter/home_mixer/federated", "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product", "home-mixer/thrift/src/main/thrift:thrift-scala", "joinkey/src/main/scala/com/twitter/joinkey/context", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter", "strato/src/main/scala/com/twitter/strato/fed/server", "timelines/src/main/scala/com/twitter/timelines/config", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.scala ================================================ package com.twitter.home_mixer import com.twitter.finatra.http.routing.HttpWarmup import com.twitter.finatra.httpclient.RequestBuilder._ import com.twitter.util.logging.Logging import com.twitter.inject.utils.Handler import com.twitter.util.Try import javax.inject.Inject import javax.inject.Singleton @Singleton class HomeMixerHttpServerWarmupHandler @Inject() (warmup: HttpWarmup) extends Handler with Logging { override def handle(): Unit = { Try(warmup.send(get("/admin/product-mixer/product-pipelines"), admin = true)()) .onFailure(e => error(e.getMessage, e)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala ================================================ package com.twitter.home_mixer import com.google.inject.Module import com.twitter.conversions.DurationOps._ import com.twitter.conversions.StorageUnitOps.richStorageUnitFromInt import com.twitter.finagle.Filter import com.twitter.finagle.Http import com.twitter.finagle.Thrift import com.twitter.finagle.ThriftMux import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.mtls.server.MtlsStackServer.MtlsHttpServerSyntax import com.twitter.finagle.netty4.param.TrackWorkerPool import com.twitter.finatra.annotations.DarkTrafficFilterType import com.twitter.finatra.http.HttpServer import com.twitter.finatra.http.routing.HttpRouter import com.twitter.finatra.mtls.http.{Mtls => HttpMtls} import com.twitter.finatra.mtls.thriftmux.Mtls import com.twitter.finatra.mtls.thriftmux.modules.MtlsThriftWebFormsModule import com.twitter.finatra.thrift.ThriftServer import com.twitter.finatra.thrift.filters._ import com.twitter.finatra.thrift.routing.ThriftRouter import com.twitter.home_mixer.controller.HomeHttpController import com.twitter.home_mixer.controller.HomeThriftController import com.twitter.home_mixer.federated.HomeMixerColumn import com.twitter.home_mixer.module._ import com.twitter.home_mixer.param.GlobalParamConfigModule import com.twitter.home_mixer.product.HomeMixerProductModule import com.twitter.home_mixer.{thriftscala => st} import com.twitter.joinkey.context.CreateRequestJoinKeyContextFilter import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule import com.twitter.product_mixer.component_library.module.CommunitiesMixerClientModule import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule import com.twitter.product_mixer.component_library.module.EarlybirdModule import com.twitter.product_mixer.component_library.module.FeedbackHistoryClientModule import com.twitter.product_mixer.component_library.module.GizmoduckClientModule import com.twitter.product_mixer.component_library.module.MemcachedImpressionBloomFilterStoreModule import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule import com.twitter.product_mixer.component_library.module.StaleTweetsCacheModule import com.twitter.product_mixer.component_library.module.TestUserMapperConfigModule import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule import com.twitter.product_mixer.component_library.module.TweetMixerClientModule import com.twitter.product_mixer.component_library.module.UserSessionStoreModule import com.twitter.product_mixer.component_library.module.UtegClientModule import com.twitter.product_mixer.component_library.module.UtvgClientModule import com.twitter.product_mixer.core.controllers.ProductMixerController import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper import com.twitter.product_mixer.core.module.ProductMixerModule import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule import com.twitter.scrooge.TUnsafeBinaryProtocolFactory import com.twitter.strato.fed.StratoFed import com.twitter.strato.fed.server.StratoFedServer object HomeMixerServerMain extends HomeMixerServer class HomeMixerServer extends StratoFedServer with ThriftServer with Mtls with HttpServer with HttpMtls { override val name = "home-mixer-server" override val modules: Seq[Module] = Seq( AccountRecommendationsMixerModule, AdvertiserBrandSafetySettingsStoreModule, ClientSentImpressionsPublisherModule, ClusterDetailsModule, CommunitiesMixerClientModule, ConversationServiceModule, EarlybirdModule, EarlybirdRealtimeCGModule, EventsRecosClientModule, FeedbackHistoryClientModule, GizmoduckClientModule, GizmoduckTimelinesCacheClientModule, GlobalParamConfigModule, HomeAdsCandidateSourceModule, HomeMixerFeaturesModule, HomeMixerFlagsModule, HomeMixerProductModule, HomeMixerResourcesModule, InMemoryCacheModule, InjectionHistoryClientModule, LimiterModule, ManhattanClientsModule, ManhattanFeatureRepositoryModule, MediaClusterId88Module, MediaClusterId95Module, MemcachedFeatureRepositoryModule, MemcachedImpressionBloomFilterStoreModule, MemcachedScoredCandidateFeaturesStoreModule, NaviModelClientModule, OnboardingTaskServiceModule, OptimizedStratoClientModule, PeopleDiscoveryServiceModule, PhoenixClientModule, ProductMixerModule, RealGraphInNetworkScoresModule, RealtimeAggregateFeatureRepositoryModule, ScoredTweetsMemcacheModule, ScoredVideoTweetsMemcacheModule, ScribeEventPublisherModule, SimClustersRecentEngagementsClientModule, SocialGraphServiceModule, StaleTweetsCacheModule, TestUserMapperConfigModule, ThriftFeatureRepositoryModule, TimelineRankerClientModule, TimelineServiceClientModule, TimelinesPersistenceStoreClientModule, TopicSocialProofClientModule, TvWatchHistoryCacheClientModule, TweetImpressionStoreModule, TweetMixerClientModule, TweetWatchTimeMetadataModule, TweetypieClientModule, TweetypieStaticEntitiesCacheClientModule, TwhinEmbeddingsModule, UserSessionStoreModule, UtegClientModule, UttTopicModule, UtvgClientModule, VideoEmbeddingModule, new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](), new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this), new ProductScopeStringCenterModule() ) val requestJoinKeyContextFilter = new CreateRequestJoinKeyContextFilter override def configureThrift(router: ThriftRouter): Unit = { router .filter[LoggingMDCFilter] .filter[TraceIdMDCFilter] .filter[ThriftMDCFilter] .filter[StatsFilter] .filter[AccessLoggingFilter] .filter[ExceptionMappingFilter] .filter[Filter.TypeAgnostic, DarkTrafficFilterType] .filter(requestJoinKeyContextFilter) .exceptionMapper[LoggingThrowableExceptionMapper] .exceptionMapper[PipelineFailureExceptionMapper] .add[HomeThriftController] } override def configureStratoThriftServer(server: ThriftMux.Server): ThriftMux.Server = { super .configureStratoThriftServer(server) .configured( TrackWorkerPool( enableTracking = true, trackingTaskPeriod = 20.milliseconds, threadDumpThreshold = 0.milliseconds )) .withMaxReusableBufferSize(1.megabyte.bytes.toInt) .withProtocolFactory(new TUnsafeBinaryProtocolFactory(Thrift.param.protocolFactory)) } override def configureHttp(router: HttpRouter): Unit = router .add( ProductMixerController[st.HomeMixer.MethodPerEndpoint]( this.injector, st.HomeMixer.ExecutePipeline ) ).add[HomeHttpController] override def configureHttpsServer(server: Http.Server): Http.Server = { val serviceIdentifier: ServiceIdentifier = injector.instance[ServiceIdentifier] server.withMutualTls(serviceIdentifier.copy(role = "home-mixer")) } override val dest: String = "/s/home-mixer/home-mixer:strato" override val columns: Seq[Class[_ <: StratoFed.Column]] = Seq(classOf[HomeMixerColumn]) override protected def warmup(): Unit = { handle[HomeMixerThriftServerWarmupHandler]() handle[HomeMixerHttpServerWarmupHandler]() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala ================================================ package com.twitter.home_mixer import com.twitter.finagle.thrift.ClientId import com.twitter.finatra.thrift.routing.ThriftWarmup import com.twitter.home_mixer.{thriftscala => st} import com.twitter.inject.utils.Handler import com.twitter.product_mixer.core.{thriftscala => pt} import com.twitter.scrooge.Request import com.twitter.scrooge.Response import com.twitter.util.Return import com.twitter.util.Throw import com.twitter.util.Try import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton @Singleton class HomeMixerThriftServerWarmupHandler @Inject() (warmup: ThriftWarmup) extends Handler with Logging { private val CurrentClientId = ClientId("thrift-warmup-client") private val SleepThreshold = 10000 // millis private val TestIds = Seq.empty private val BaseClientContext = pt.ClientContext( userId = TestIds.headOption, guestId = None, appId = Some(1L), ipAddress = Some("0.0.0.0"), userAgent = Some("FAKE_USER_AGENT_FOR_WARMUPS"), countryCode = Some("US"), languageCode = Some("en"), isTwoffice = None, userRoles = None, deviceId = Some("FAKE_DEVICE_ID_FOR_WARMUPS") ) def handle(): Unit = { try { CurrentClientId.asCurrent { TestIds.foreach { id => val warmupReqs = warmupQuery(id) warmupReqs.foreach { warmupReq => info(s"Sending warm-up request to service with query: $warmupReq") warmup.sendRequest( method = st.HomeMixer.GetUrtResponse, req = Request(st.HomeMixer.GetUrtResponse.Args(warmupReq)), )(assertWarmupResponse) } } } } catch { case e: Throwable => error(e.getMessage, e) } info("Warm-up done.") } private def warmupQuery(userId: Long): Seq[st.HomeMixerRequest] = { val clientContext = BaseClientContext.copy(userId = Some(userId)) val scoredTweets = st.HomeMixerRequest( clientContext = clientContext, product = st.Product.ScoredTweets, productContext = Some(st.ProductContext.ScoredTweets(st.ScoredTweets())), ) val scoredVideoTweets = st.HomeMixerRequest( clientContext = clientContext, product = st.Product.ScoredVideoTweets, productContext = Some(st.ProductContext.ScoredVideoTweets(st.ScoredVideoTweets())), ) val forYou = st.HomeMixerRequest( clientContext = clientContext, product = st.Product.ForYou, productContext = Some(st.ProductContext.ForYou(st.ForYou())), ) val following = st.HomeMixerRequest( clientContext = clientContext, product = st.Product.Following, productContext = Some(st.ProductContext.Following(st.Following())), ) Seq(scoredTweets, scoredVideoTweets, forYou, following) } private def assertWarmupResponse( result: Try[Response[st.HomeMixer.GetUrtResponse.SuccessType]] ): Unit = { result match { case Return(_) => // ok case Throw(exception) => warn("Error performing warm-up request.") error(exception.getMessage, exception) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/stale_tweets", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/transformer/stale_tweets", ], exports = [ "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.candidate_pipeline import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetTypeMetricsFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter import com.twitter.home_mixer.functional_component.filter.LocationFilter import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter import com.twitter.home_mixer.functional_component.filter.TweetHydrationFilter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata import com.twitter.product_mixer.component_library.feature_hydrator.candidate.communities.CommunityNamesFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedBulkCandidateFeatureHydrator import com.twitter.product_mixer.component_library.filter.FeatureFilter import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.BaseGate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.functional_component.transformer.DependentCandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig /** * Candidate Pipeline Config that fetches tweets from the Conversation Service Candidate Source */ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery]( conversationServiceCandidateSource: ConversationServiceCandidateSource, tweetypieFeatureHydrator: TweetypieFeatureHydrator, namesFeatureHydrator: NamesFeatureHydrator, communityNamesFeatureHydrator: CommunityNamesFeatureHydrator, invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, override val gates: Seq[BaseGate[Query]], override val decorator: Option[CandidateDecorator[Query, TweetCandidate]], servedType: hmt.ServedType, paramGatedPostContextFeatureHydrator: ParamGatedBulkCandidateFeatureHydrator[Query, TweetCandidate]) extends DependentCandidatePipelineConfig[ Query, ConversationServiceCandidateSourceRequest, TweetWithConversationMetadata, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ConversationService") private val InNetworkFilterId = "InNetwork" private val QuotedTweetDroppedFilterId = "QuotedTweetDropped" override val candidateSource: BaseCandidateSource[ ConversationServiceCandidateSourceRequest, TweetWithConversationMetadata ] = conversationServiceCandidateSource override val queryTransformer: DependentCandidatePipelineQueryTransformer[ Query, ConversationServiceCandidateSourceRequest ] = { (_, candidates) => val tweetsWithConversationMetadata = candidates.collect { case candidate if candidate.isCandidateType[TweetCandidate]() => TweetWithConversationMetadata( tweetId = candidate.candidateIdLong, userId = candidate.features.getOrElse(AuthorIdFeature, None), sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None), inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None), conversationId = None, ancestors = Seq.empty ) } ConversationServiceCandidateSourceRequest(tweetsWithConversationMetadata) } override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[TweetWithConversationMetadata] ] = Seq(ConversationServiceResponseFeatureTransformer(servedType)) override val resultTransformer: CandidatePipelineResultsTransformer[ TweetWithConversationMetadata, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[Query, TweetCandidate, _] ] = Seq( tweetypieFeatureHydrator, InNetworkFeatureHydrator, ) override def filters: Seq[Filter[Query, TweetCandidate]] = Seq( RetweetDeduplicationFilter, FeatureFilter.fromFeature(FilterIdentifier(InNetworkFilterId), InNetworkFeature), TweetHydrationFilter, PredicateFeatureFilter.fromPredicate( FilterIdentifier(QuotedTweetDroppedFilterId), shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } ), LocationFilter, invalidSubscriptionTweetFilter, InvalidConversationModuleFilter ) override val postFilterFeatureHydration: Seq[ BaseCandidateFeatureHydrator[Query, TweetCandidate, _] ] = Seq( communityNamesFeatureHydrator, namesFeatureHydrator, TweetTypeMetricsFeatureHydrator, paramGatedPostContextFeatureHydrator ) override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala ================================================ package com.twitter.home_mixer.candidate_pipeline import com.twitter.home_mixer.functional_component.decorator.HomeConversationServiceCandidateDecorator import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeTweetContextBuilder import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.PostContextFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource import com.twitter.product_mixer.component_library.feature_hydrator.candidate.communities.CommunityNamesFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedBulkCandidateFeatureHydrator import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.timelines.configapi.Param import com.twitter.product_mixer.core.pipeline.PipelineQuery import javax.inject.Inject import javax.inject.Singleton @Singleton class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() ( conversationServiceCandidateSource: ConversationServiceCandidateSource, tweetypieFeatureHydrator: TweetypieFeatureHydrator, communityNamesFeatureHydrator: CommunityNamesFeatureHydrator, postContextFeatureHydrator: PostContextFeatureHydrator, invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, namesFeatureHydrator: NamesFeatureHydrator, homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, homeTweetContextBuilder: HomeTweetContextBuilder) { def build( nonEmptyCandidateScope: CandidateScope, servedType: hmt.ServedType, enablePostContextFeatureHydratorParam: Param[Boolean] ): ConversationServiceCandidatePipelineConfig[Query] = { val paramGatedPostContextFeatureHydrator = ParamGatedBulkCandidateFeatureHydrator( enablePostContextFeatureHydratorParam, postContextFeatureHydrator ) new ConversationServiceCandidatePipelineConfig( conversationServiceCandidateSource, tweetypieFeatureHydrator, namesFeatureHydrator, communityNamesFeatureHydrator, invalidSubscriptionTweetFilter, Seq(NonEmptyCandidatesGate(nonEmptyCandidateScope)), HomeConversationServiceCandidateDecorator( homeFeedbackActionInfoBuilder, homeTweetContextBuilder, servedType ), servedType, paramGatedPostContextFeatureHydrator ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.candidate_pipeline import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier case class ConversationServiceResponseFeatureTransformer(servedType: hmt.ServedType) extends CandidateFeatureTransformer[TweetWithConversationMetadata] { override val identifier: TransformerIdentifier = TransformerIdentifier("ConversationServiceResponse") override val features: Set[Feature[_, _]] = Set( AuthorIdFeature, InReplyToTweetIdFeature, IsRetweetFeature, SourceTweetIdFeature, SourceUserIdFeature, ConversationModuleFocalTweetIdFeature, AncestorsFeature, ServedTypeFeature ) override def transform(candidate: TweetWithConversationMetadata): FeatureMap = FeatureMapBuilder() .add(AuthorIdFeature, candidate.userId) .add(InReplyToTweetIdFeature, candidate.inReplyToTweetId) .add(IsRetweetFeature, candidate.sourceTweetId.isDefined) .add(SourceTweetIdFeature, candidate.sourceTweetId) .add(SourceUserIdFeature, candidate.sourceUserId) .add(ConversationModuleFocalTweetIdFeature, candidate.conversationId) .add(AncestorsFeature, candidate.ancestors) .add(ServedTypeFeature, servedType) .build() } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.candidate_pipeline import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.candidate_source.stale_tweets.StaleTweetsCacheCandidateSource import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.EmptyClientEventInfoBuilder import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.transformer.stale_tweets.EditedTweetsCandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineFocalTweetSafetyLevel import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig import javax.inject.Inject import javax.inject.Singleton /** * Candidate Pipeline Config that fetches edited tweets from the Stale Tweets Cache */ @Singleton case class EditedTweetsCandidatePipelineConfig @Inject() ( staleTweetsCacheCandidateSource: StaleTweetsCacheCandidateSource, namesFeatureHydrator: NamesFeatureHydrator, homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) extends DependentCandidatePipelineConfig[ PipelineQuery, Seq[Long], Long, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("EditedTweets") override val candidateSource: BaseCandidateSource[Seq[Long], Long] = staleTweetsCacheCandidateSource override val queryTransformer: CandidatePipelineQueryTransformer[PipelineQuery, Seq[Long]] = EditedTweetsCandidatePipelineQueryTransformer(PersistenceEntriesFeature) override val resultTransformer: CandidatePipelineResultsTransformer[Long, TweetCandidate] = { candidate => TweetCandidate(id = candidate) } override val postFilterFeatureHydration: Seq[ BaseCandidateFeatureHydrator[PipelineQuery, TweetCandidate, _] ] = Seq(namesFeatureHydrator) override val decorator: Option[CandidateDecorator[PipelineQuery, TweetCandidate]] = { val tweetItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( clientEventInfoBuilder = EmptyClientEventInfoBuilder, entryIdToReplaceBuilder = Some((_, candidate, _) => Some(s"${TweetItem.TweetEntryNamespace}-${candidate.id.toString}")), contextualTweetRefBuilder = Some( ContextualTweetRefBuilder( TweetHydrationContext( // Apply safety level that includes canonical VF treatments that apply regardless of context. safetyLevelOverride = Some(TimelineFocalTweetSafetyLevel), outerTweetContext = None ) ) ), feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) ) Some(UrtItemCandidateDecorator(tweetItemBuilder)) } override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.5, 50, 60, 60)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/NewTweetsPillCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.candidate_pipeline import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.functional_component.gate.RequestContextNotGate import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature import com.twitter.home_mixer.model.request.DeviceContext import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.DurationParamBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.ShowAlertCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.StaticShowAlertColorConfigurationBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.StaticShowAlertDisplayLocationBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.alert.StaticShowAlertIconDisplayInfoBuilder import com.twitter.product_mixer.component_library.gate.FeatureGate import com.twitter.product_mixer.component_library.model.candidate.ShowAlertCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.candidate_source.StaticCandidateSource import com.twitter.product_mixer.core.functional_component.configapi.StaticParam import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.alert.BaseDurationBuilder import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.NewTweets import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.ShowAlertColorConfiguration import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.ShowAlertIconDisplayInfo import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.Top import com.twitter.product_mixer.core.model.marshalling.response.urt.alert.UpArrow import com.twitter.product_mixer.core.model.marshalling.response.urt.color.TwitterBlueRosettaColor import com.twitter.product_mixer.core.model.marshalling.response.urt.color.WhiteRosettaColor import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig import com.twitter.util.Duration import javax.inject.Inject import javax.inject.Singleton /** * Candidate Pipeline Config that creates the New Tweets Pill */ @Singleton class NewTweetsPillCandidatePipelineConfig[Query <: PipelineQuery with HasDeviceContext] @Inject() ( ) extends DependentCandidatePipelineConfig[ Query, Unit, ShowAlertCandidate, ShowAlertCandidate ] { import NewTweetsPillCandidatePipelineConfig._ override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("NewTweetsPill") override val gates: Seq[Gate[Query]] = Seq( RequestContextNotGate(Seq(DeviceContext.RequestContext.PullToRefresh)), FeatureGate.fromFeature(GetNewerFeature) ) override val candidateSource: CandidateSource[Unit, ShowAlertCandidate] = StaticCandidateSource( CandidateSourceIdentifier(identifier.name), Seq(ShowAlertCandidate(id = identifier.name, userIds = Seq.empty)) ) override val queryTransformer: CandidatePipelineQueryTransformer[Query, Unit] = { _ => Unit } override val resultTransformer: CandidatePipelineResultsTransformer[ ShowAlertCandidate, ShowAlertCandidate ] = { candidate => candidate } override val decorator: Option[CandidateDecorator[Query, ShowAlertCandidate]] = { val triggerDelayBuilder = new BaseDurationBuilder[Query] { override def apply( query: Query, candidate: ShowAlertCandidate, features: FeatureMap ): Option[Duration] = { val delay = query.deviceContext.flatMap(_.requestContextValue) match { case Some(DeviceContext.RequestContext.TweetSelfThread) => 0.millis case Some(DeviceContext.RequestContext.ManualRefresh) => 0.millis case _ => TriggerDelay } Some(delay) } } val homeShowAlertCandidateBuilder = ShowAlertCandidateUrtItemBuilder( alertType = NewTweets, colorConfigBuilder = StaticShowAlertColorConfigurationBuilder(DefaultColorConfig), displayLocationBuilder = StaticShowAlertDisplayLocationBuilder(Top), triggerDelayBuilder = Some(triggerDelayBuilder), displayDurationBuilder = Some(DurationParamBuilder(StaticParam(DisplayDuration))), iconDisplayInfoBuilder = Some(StaticShowAlertIconDisplayInfoBuilder(DefaultIconDisplayInfo)) ) Some(UrtItemCandidateDecorator(homeShowAlertCandidateBuilder)) } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() ) } object NewTweetsPillCandidatePipelineConfig { val DefaultColorConfig: ShowAlertColorConfiguration = ShowAlertColorConfiguration( background = TwitterBlueRosettaColor, text = WhiteRosettaColor, border = Some(WhiteRosettaColor) ) val DefaultIconDisplayInfo: ShowAlertIconDisplayInfo = ShowAlertIconDisplayInfo(icon = UpArrow, tint = WhiteRosettaColor) // Unlimited display time (until user takes action) val DisplayDuration = -1.millisecond val TriggerDelay = 4.minutes } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/TimelineServiceResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.candidate_pipeline import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.timelineservice.{thriftscala => t} object TimelineServiceResponseFeatureTransformer extends CandidateFeatureTransformer[t.Tweet] { override val identifier: TransformerIdentifier = TransformerIdentifier("TimelineServiceResponse") override val features: Set[Feature[_, _]] = Set( AuthorIdFeature, InReplyToTweetIdFeature, IsRetweetFeature, SourceTweetIdFeature, SourceUserIdFeature, ) override def transform(candidate: t.Tweet): FeatureMap = FeatureMapBuilder() .add(AuthorIdFeature, candidate.userId) .add(InReplyToTweetIdFeature, candidate.inReplyToStatusId) .add(IsRetweetFeature, candidate.sourceStatusId.isDefined) .add(SourceTweetIdFeature, candidate.sourceStatusId) .add(SourceUserIdFeature, candidate.sourceUserId) .build() } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/VerifiedPromptCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.candidate_pipeline import com.twitter.home_mixer.functional_component.decorator.builder.VerifiedPromptBuilder import com.twitter.home_mixer.functional_component.gate.RateLimitNotGate import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.model.candidate.InlinePromptCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.candidate_source.IdentityCandidateExtractor import com.twitter.product_mixer.core.functional_component.candidate_source.PassthroughCandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.ExternalStringRegistry import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton class VerifiedPromptCandidatePipelineConfig @Inject() ( identityCandidateExtractor: IdentityCandidateExtractor[PipelineQuery], @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry], @ProductScoped stringCenterProvider: Provider[StringCenter]) extends DependentCandidatePipelineConfig[ PipelineQuery, PipelineQuery, PipelineQuery, InlinePromptCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("VerifiedPrompt") override val gates: Seq[Gate[PipelineQuery]] = Seq(RateLimitNotGate) override def candidateSource: CandidateSource[ PipelineQuery, PipelineQuery ] = PassthroughCandidateSource( CandidateSourceIdentifier("VerifiedPassthroughCandidateSource"), identityCandidateExtractor ) override val queryTransformer: CandidatePipelineQueryTransformer[PipelineQuery, PipelineQuery] = identity override val resultTransformer: CandidatePipelineResultsTransformer[ PipelineQuery, InlinePromptCandidate ] = { query => InlinePromptCandidate(id = query.getRequiredUserId.toString) } override val decorator: Option[ CandidateDecorator[PipelineQuery, InlinePromptCandidate] ] = { val clientEventInfoBuilder = ClientEventInfoBuilder[PipelineQuery, InlinePromptCandidate]("verified_prompt") val stringCenter = stringCenterProvider.get() val externalStringRegistry = externalStringRegistryProvider.get() val verifiedPromptBuilder = VerifiedPromptBuilder( clientEventInfoBuilder, stringCenter, externalStringRegistry ) Some(UrtItemCandidateDecorator(verifiedPromptBuilder)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "finatra/http-server/src/main/scala/com/twitter/finatra/http", "finatra/thrift/src/main/scala/com/twitter/finatra/thrift:controller", "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt", "snowflake/src/main/scala/com/twitter/snowflake/id", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeHttpController.scala ================================================ package com.twitter.home_mixer.controller import com.twitter.finagle.http.Request import com.twitter.finatra.http.Controller import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.model.request.ScoredTweetsProduct import com.twitter.home_mixer.model.request.ScoredTweetsProductContext import com.twitter.home_mixer.service.ScoredTweetsService import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder import com.twitter.product_mixer.core.model.marshalling.request.ClientContext import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.Params import com.twitter.util.jackson.ScalaObjectMapper import javax.inject.Inject import javax.inject.Singleton @Singleton class HomeHttpController @Inject() ( scoredTweetsService: ScoredTweetsService, paramsBuilder: ParamsBuilder, mapper: ScalaObjectMapper) extends Controller { private val UserIdParam = "userId" private val CountryCode = "US" private val LanguageCode = "en" private val AppId = 1L private val DefaultUserId = 1L private val UserAgent = "" private val BaseClientContext = ClientContext( userId = Some(DefaultUserId), guestId = None, appId = Some(AppId), ipAddress = None, userAgent = Some(UserAgent), countryCode = Some(CountryCode), languageCode = Some(LanguageCode), isTwoffice = None, userRoles = None, deviceId = None, mobileDeviceId = None, mobileDeviceAdId = None, limitAdTracking = None, guestIdAds = None, guestIdMarketing = None, authenticatedUserId = None, isVerifiedCrawler = None ) case class ScoredTweetMetadata(tweetId: Long, authorId: Long, inNetwork: Boolean, text: String) get("/scoredTweets") { request: Request => val userId = request.getLongParam(UserIdParam) val hmRequest = HomeMixerRequest( clientContext = BaseClientContext.copy(userId = Some(userId)), product = ScoredTweetsProduct, productContext = Some(ScoredTweetsProductContext(None, None, None, None, None, None, None, None)), serializedRequestCursor = None, maxResults = None, debugParams = None, homeRequestParam = false ) val params = buildParams(hmRequest) val response = scoredTweetsService.getScoredTweetsResponse[HomeMixerRequest](hmRequest, params) Stitch.run(response).map { scoredTweetsResponse => scoredTweetsResponse.scoredTweets.map { tweet => val tweetData = ScoredTweetMetadata( tweet.tweetId, tweet.authorId, tweet.inNetwork.getOrElse(false), tweet.tweetText.getOrElse("") ) mapper.writeValueAsString(tweetData) } } } private def buildParams(request: HomeMixerRequest): Params = { val userAgeOpt = request.clientContext.userId.map { userId => SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue) } val fsCustomMapInput = userAgeOpt.map("account_age_in_days" -> _).toMap paramsBuilder.build( clientContext = request.clientContext, product = request.product, featureOverrides = request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty), fsCustomMapInput = fsCustomMapInput ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala ================================================ package com.twitter.home_mixer.controller import com.twitter.finatra.thrift.Controller import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.service.ScoredTweetsService import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.core.controllers.DebugTwitterContext import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder import com.twitter.product_mixer.core.service.debug_query.DebugQueryService import com.twitter.product_mixer.core.service.urt.UrtService import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.Params import javax.inject.Inject import server.src.main.scala.com.twitter.home_mixer.service.HeavyRankerScoresService class HomeThriftController @Inject() ( homeRequestUnmarshaller: HomeMixerRequestUnmarshaller, debugQueryService: DebugQueryService, urtService: UrtService, scoredTweetsService: ScoredTweetsService, heavyRankerScoresService: HeavyRankerScoresService, paramsBuilder: ParamsBuilder) extends Controller(t.HomeMixer) with DebugTwitterContext { handle(t.HomeMixer.GetUrtResponse) { args: t.HomeMixer.GetUrtResponse.Args => val request = homeRequestUnmarshaller(args.request) val params = buildParams(request) Stitch.run(urtService.getUrtResponse[HomeMixerRequest](request, params)) } handle(t.HomeMixer.DebugGetUrtResponse) { args: t.HomeMixer.DebugGetUrtResponse.Args => val request = homeRequestUnmarshaller(args.request) val params = buildParams(request) withDebugTwitterContext(request.clientContext) { Stitch.run(urtService.getUrtResponse[HomeMixerRequest](request, params)) } } // Handle debug requests handle(t.HomeMixer.ExecutePipeline) .withService(debugQueryService(homeRequestUnmarshaller.apply)) handle(t.HomeMixer.GetScoredTweetsResponse) { args: t.HomeMixer.GetScoredTweetsResponse.Args => val request = homeRequestUnmarshaller(args.request) val params = buildParams(request) withDebugTwitterContext(request.clientContext) { Stitch.run(scoredTweetsService.getScoredTweetsResponse[HomeMixerRequest](request, params)) } } handle(t.HomeMixer.GetHeavyRankerScoresResponse) { args: t.HomeMixer.GetHeavyRankerScoresResponse.Args => val request = homeRequestUnmarshaller(args.request) val params = buildParams(request) withDebugTwitterContext(request.clientContext) { Stitch.run( heavyRankerScoresService.getHeavyRankerScoresResponse[HomeMixerRequest](request, params)) } } private def buildParams(request: HomeMixerRequest): Params = { val userAgeOpt = request.clientContext.userId.map { userId => // Setting to Int.MaxValue for cases where id is not snowflake id as they are pretty old accounts SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue) } val fsCustomMapInput = userAgeOpt.map("account_age_in_days" -> _).toMap paramsBuilder.build( clientContext = request.clientContext, product = request.product, featureOverrides = request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty), fsCustomMapInput = fsCustomMapInput ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/thrift/src/main/thrift:thrift-scala", "joinkey/src/main/scala/com/twitter/joinkey/context", "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", "stitch/stitch-gizmoduck", "strato/config/columns/auth-context:auth-context-strato-client", "strato/src/main/scala/com/twitter/strato/callcontext", "strato/src/main/scala/com/twitter/strato/fed/server", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala ================================================ package com.twitter.home_mixer.federated import com.twitter.conversions.DurationOps._ import com.twitter.finagle.filter.SLODefinition import com.twitter.finagle.filter.SLOStatsFilter import com.twitter.finagle.service.ResponseClassifier import com.twitter.finagle.stats.DefaultStatsReceiver import com.twitter.gizmoduck.{thriftscala => gd} import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.{thriftscala => hm} import com.twitter.joinkey.context.thriftscala.{RequestJoinKeyContext => ThriftJoinKeyContext} import com.twitter.joinkey.context.RequestJoinKeyContext import com.twitter.joinkey.context.ThreadedLocalJoinKeyRandomGenerator import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry import com.twitter.product_mixer.core.{thriftscala => pm} import com.twitter.stitch.Arrow import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch.Letter import com.twitter.stitch.gizmoduck.Gizmoduck import com.twitter.strato.callcontext.CallContext import com.twitter.strato.catalog.OpMetadata import com.twitter.strato.config._ import com.twitter.strato.data._ import com.twitter.strato.fed.StratoFed import com.twitter.strato.generated.client.auth_context.AuditIpClientColumn import com.twitter.strato.graphql.timelines.{thriftscala => gql} import com.twitter.strato.thrift.ScroogeConv import com.twitter.timelines.render.thriftscala.Timeline import com.twitter.timelines.render.{thriftscala => tr} import com.twitter.util.Try import javax.inject.Inject import javax.inject.Singleton @Singleton class HomeMixerColumn @Inject() ( homeMixerRequestUnmarshaller: HomeMixerRequestUnmarshaller, gizmoduck: Gizmoduck, auditIpClientColumn: AuditIpClientColumn, paramsBuilder: ParamsBuilder, productPipelineRegistry: ProductPipelineRegistry) extends StratoFed.Column(HomeMixerColumn.Path) with StratoFed.Fetch.Arrow { override val contactInfo: ContactInfo = ContactInfo( contactEmail = "", ldapGroup = "", jiraProject = "", slackRoomId = "" ) override val metadata: OpMetadata = OpMetadata( lifecycle = Some(Lifecycle.Production), description = Some(Description.PlainText("Federated Strato column for Timelines served via Home Mixer")) ) private val bouncerAccess: Seq[Policy] = Seq(BouncerAccess()) private val finatraTestServiceIdentifiers: Seq[Policy] = Seq( ServiceIdentifierPattern( role = "", service = "", env = "", zone = Seq("")) ) private val sloStatsFilter = { val requestToSLODefinition: PartialFunction[Any, SLODefinition] = { case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) => SLODefinition("ForYou", 1000.millis) } new SLOStatsFilter[gql.TimelineKey, Result[Timeline]]( requestToSLODefinition = requestToSLODefinition, responseClassifier = ResponseClassifier.Default, statsReceiver = DefaultStatsReceiver.scope("slo") ) } override val policy: Policy = AnyOf(bouncerAccess ++ finatraTestServiceIdentifiers) override type Key = gql.TimelineKey override type View = gql.HomeTimelineView override type Value = tr.Timeline override val keyConv: Conv[Key] = ScroogeConv.fromStruct[gql.TimelineKey] override val viewConv: Conv[View] = ScroogeConv.fromStruct[gql.HomeTimelineView] override val valueConv: Conv[Value] = ScroogeConv.fromStruct[tr.Timeline] // For populating requestJoinId private val joinIdGenerator = new ThreadedLocalJoinKeyRandomGenerator // For populating user roles private val queryFieldsRoles = gd.QueryFields.Roles private object RequestJoinKeyLetter extends Letter[Option[Long]] { override def let[A](requestJoinId: Option[Long])(fn: => A): A = RequestJoinKeyContext.let(ThriftJoinKeyContext(requestJoinId))(fn) } private def createHomeMixerRequestArrow( auditIpClientColumn: AuditIpClientColumn, ): Arrow[(Key, View), hm.HomeMixerRequest] = { val populateUserRolesAndIp: Arrow[(Key, View), (Option[Set[String]], Option[String])] = { val populateUserRoles = Arrow .flatMap[(Key, View), Option[Set[String]]] { _ => Stitch.collect { CallContext.twitterUserId.map { userId => gizmoduck .getUserById( userId = userId, queryFields = Set(queryFieldsRoles), context = gd.LookupContext(forUserId = Some(userId)) ).map(_.roles.map(_.roles.toSet).getOrElse(Set.empty)) } } } val populateIpAddress = Arrow .flatMap[(Key, View), Option[String]](_ => auditIpClientColumn.fetcher .callStack(HomeMixerColumn.FetchCallstack) .fetch((), ()).map(_.v)) Arrow.join( populateUserRoles, populateIpAddress ) } Arrow.zipWithArg(populateUserRolesAndIp).map { case ((key, view), (roles, ipAddress)) => val deviceContextOpt = Some( hm.DeviceContext( isPolling = CallContext.isPolling, requestContext = view.requestContext, latestControlAvailable = view.latestControlAvailable, autoplayEnabled = view.autoplayEnabled )) val seenTweetIds = view.seenTweetIds.filter(_.nonEmpty) val (product, productContext) = key match { case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) => ( hm.Product.ForYou, hm.ProductContext.ForYou( hm.ForYou( deviceContextOpt, seenTweetIds, view.dspClientContext, view.pushToHomeTweetId ) )) case gql.TimelineKey.HomeLatestTimeline(_) | gql.TimelineKey.HomeLatestTimelineV2(_) => ( hm.Product.Following, hm.ProductContext.Following( hm.Following(deviceContextOpt, seenTweetIds, view.dspClientContext))) case gql.TimelineKey.CreatorSubscriptionsTimeline(_) => ( hm.Product.Subscribed, hm.ProductContext.Subscribed(hm.Subscribed(deviceContextOpt, seenTweetIds))) case _ => throw new UnsupportedOperationException(s"Unknown product: $key") } val clientContext = pm.ClientContext( userId = CallContext.twitterUserId, guestId = CallContext.guestId, guestIdAds = CallContext.guestIdAds, guestIdMarketing = CallContext.guestIdMarketing, appId = CallContext.clientApplicationId, ipAddress = ipAddress, userAgent = CallContext.userAgent, countryCode = CallContext.requestCountryCode, languageCode = CallContext.requestLanguageCode, isTwoffice = CallContext.isInternalOrTwoffice, userRoles = roles, deviceId = CallContext.deviceId, mobileDeviceId = CallContext.mobileDeviceId, mobileDeviceAdId = CallContext.adId, limitAdTracking = CallContext.limitAdTracking, authenticatedUserId = CallContext.authenticatedTwitterUserId, isVerifiedCrawler = CallContext.isVerifiedCrawler ) hm.HomeMixerRequest( clientContext = clientContext, product = product, productContext = Some(productContext), maxResults = Try(view.count.get.toInt).toOption.orElse(HomeMixerColumn.MaxCount), cursor = view.cursor.filter(_.nonEmpty) ) } } override val fetch: Arrow[(Key, View), Result[Value]] = { val transformThriftIntoPipelineRequest: Arrow[ (Key, View), ProductPipelineRequest[HomeMixerRequest] ] = { Arrow .identity[(Key, View)] .andThen { createHomeMixerRequestArrow(auditIpClientColumn) } .map { thriftRequest => val request = homeMixerRequestUnmarshaller(thriftRequest) val params = paramsBuilder.build( clientContext = request.clientContext, product = request.product, featureOverrides = request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty), ) ProductPipelineRequest(request, params) } } val underlyingProduct: Arrow[ ProductPipelineRequest[HomeMixerRequest], ProductPipelineResult[tr.TimelineResponse] ] = Arrow .identity[ProductPipelineRequest[HomeMixerRequest]] .map { pipelineRequest => val pipelineArrow = Arrow.let(RequestJoinKeyLetter)(joinIdGenerator.get().getNewRequestJoinId) { // Populate requestJoinId productPipelineRegistry .getProductPipeline[HomeMixerRequest, tr.TimelineResponse]( pipelineRequest.request.product) .arrow } (pipelineArrow, pipelineRequest) }.applyArrow Arrow .zipWithArg( Arrow.time { transformThriftIntoPipelineRequest .andThen(underlyingProduct) .map { _.result match { case Some(result) => found(result.timeline) case _ => missing } } } ).map { case ((key, _), (result, duration)) => sloStatsFilter.record(key, result, duration.inNanoseconds) result }.lowerFromTry } } object HomeMixerColumn { val Path = "home-mixer/homeMixer.Timeline" private val FetchCallstack = s"$Path:fetch" private val MaxCount: Option[Int] = Some(100) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/javax/inject:javax.inject", "finagle/finagle-memcached/src/main/scala", "finatra/inject/inject-core/src/main/scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", "src/thrift/com/twitter/search:earlybird-scala", "stitch/stitch-timelineservice/src/main/scala", ], exports = [ "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/EarlybirdCandidateSource.scala ================================================ package com.twitter.home_mixer.functional_component.candidate_source import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures import com.twitter.product_mixer.core.functional_component.candidate_source.CandidatesWithSourceFeatures import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.search.earlybird.{thriftscala => t} import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton case object EarlybirdResponseTruncatedFeature extends FeatureWithDefaultOnFailure[t.EarlybirdRequest, Boolean] { override val defaultValue: Boolean = false } case object EarlybirdBottomTweetFeature extends FeatureWithDefaultOnFailure[t.EarlybirdRequest, Option[Long]] { override val defaultValue: Option[Long] = None } @Singleton case class EarlybirdCandidateSource @Inject() ( earlybird: t.EarlybirdService.MethodPerEndpoint) extends CandidateSourceWithExtractedFeatures[t.EarlybirdRequest, t.ThriftSearchResult] { override val identifier = CandidateSourceIdentifier("Earlybird") override def apply( request: t.EarlybirdRequest ): Stitch[CandidatesWithSourceFeatures[t.ThriftSearchResult]] = { Stitch.callFuture(earlybird.search(request)).map { response => val candidates = response.searchResults.map(_.results).getOrElse(Seq.empty) val features = FeatureMapBuilder() .add(EarlybirdResponseTruncatedFeature, candidates.size == request.searchQuery.numResults) .add(EarlybirdBottomTweetFeature, candidates.lastOption.map(_.id)) .build() CandidatesWithSourceFeatures(candidates, features) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/StaleTweetsCacheCandidateSource.scala ================================================ package com.twitter.home_mixer.functional_component.candidate_source import com.google.inject.name.Named import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.home_mixer.param.HomeMixerInjectionNames.StaleTweetsCache import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class StaleTweetsCacheCandidateSource @Inject() ( @Named(StaleTweetsCache) staleTweetsCache: MemcachedClient) extends CandidateSource[Seq[Long], Long] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("StaleTweetsCache") private val StaleTweetsCacheKeyPrefix = "v1_" override def apply(request: Seq[Long]): Stitch[Seq[Long]] = { val keys = request.map(StaleTweetsCacheKeyPrefix + _) Stitch.callFuture(staleTweetsCache.get(keys).map { tweets => tweets.map { case (k, _) => k.replaceFirst(StaleTweetsCacheKeyPrefix, "").toLong }.toSeq }) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/pivot", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/trends_events", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/EntryPointPivotModuleDecorator.scala ================================================ package com.twitter.home_mixer.functional_component.decorator import com.twitter.home_mixer.functional_component.decorator.EntryPointPivotModuleCandidateDecorator.EntryNamespaceString import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.UrtItemInModuleDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.pivot.PivotCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.SuggestTypeClientEventDetailsBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleHeaderBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder import com.twitter.product_mixer.component_library.model.candidate.pivot.PivotCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseStr import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Vertical import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelineservice.suggests.{thriftscala => st} case class StrEntryPointPivotCategoryText( defaultText: String) extends BaseStr[PipelineQuery, PivotCandidate] { def apply( query: PipelineQuery, candidate: PivotCandidate, candidateFeatures: FeatureMap ): String = candidate.categoryText.getOrElse(defaultText) } object EntryPointPivotModuleCandidateDecorator { val EntryNamespaceString = "entry-point-pivot" } case class EntryPointPivotModuleCandidateDecorator( component: String, headerText: BaseStr[PipelineQuery, PivotCandidate]) { private val clientEventsBuilder = ClientEventInfoBuilder[PipelineQuery, PivotCandidate](component) private val clientEventDetailsBuilder = SuggestTypeClientEventDetailsBuilder(st.SuggestType.EntryPointPivot) private val itemBuilder = PivotCandidateUrtItemBuilder(clientEventInfoBuilder = Some(clientEventDetailsBuilder)) private val itemDecorator = UrtItemCandidateDecorator(itemBuilder) private val moduleHeaderBuilder = ModuleHeaderBuilder( textBuilder = headerText, isSticky = Some(false), urlBuilder = None ) private val moduleBuilder = TimelineModuleBuilder( entryNamespace = EntryNamespace(EntryNamespaceString), displayTypeBuilder = StaticModuleDisplayTypeBuilder(Vertical), clientEventInfoBuilder = clientEventsBuilder, headerBuilder = Some(moduleHeaderBuilder), footerBuilder = None ) val moduleDecorator: UrtItemInModuleDecorator[PipelineQuery, PivotCandidate, Nothing] = UrtItemInModuleDecorator(itemDecorator, moduleBuilder) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ForYouTweetCandidateDecorator.scala ================================================ package com.twitter.home_mixer.functional_component.decorator import com.twitter.home_mixer.functional_component.decorator.ForYouTweetCandidateDecorator.ConvoModuleEntryNamespace import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeTweetContextBuilder import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeTweetSocialContextBuilder import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ManualModuleId import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation import com.twitter.product_mixer.core.pipeline.PipelineQuery import javax.inject.Inject import javax.inject.Singleton object ForYouTweetCandidateDecorator { val ConvoModuleEntryNamespace = EntryNamespace("home-conversation") } @Singleton class ForYouTweetCandidateDecorator @Inject() ( homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, homeTweetSocialContextBuilder: HomeTweetSocialContextBuilder, homeTweetContextBuilder: HomeTweetContextBuilder) { val clientEventInfoBuilder = HomeClientEventInfoBuilder() val tweetItemBuilder = TweetCandidateUrtItemBuilder( clientEventInfoBuilder = clientEventInfoBuilder, socialContextBuilder = Some(homeTweetSocialContextBuilder), timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder), feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder), tweetContext = Some(homeTweetContextBuilder) ) val tweetDecorator = UrtItemCandidateDecorator(tweetItemBuilder) val moduleBuilder = TimelineModuleBuilder( entryNamespace = ConvoModuleEntryNamespace, clientEventInfoBuilder = clientEventInfoBuilder, moduleIdGeneration = ManualModuleId(0L), displayTypeBuilder = StaticModuleDisplayTypeBuilder(VerticalConversation), metadataBuilder = Some(HomeConversationModuleMetadataBuilder()) ) val decorator: UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long] = UrtMultipleModulesDecorator( urtItemCandidateDecorator = tweetDecorator, moduleBuilder = moduleBuilder, groupByKey = (_, _, candidateFeatures) => candidateFeatures.getOrElse(ConversationModuleFocalTweetIdFeature, None) ) def build(): UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long] = decorator } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala ================================================ package com.twitter.home_mixer.functional_component.decorator import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder import com.twitter.home_mixer.functional_component.decorator.urt.builder.{HomeFeedbackActionInfoBuilder, HomeTweetContextBuilder} import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.social_context.CommunitiesSocialContextBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation import com.twitter.product_mixer.core.pipeline.PipelineQuery object HomeConversationServiceCandidateDecorator { private val ConversationModuleNamespace = EntryNamespace("home-conversation") def apply( homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, homeTweetContextBuilder: HomeTweetContextBuilder, servedType: hmt.ServedType ): Some[UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long]] = { val component = servedType.originalName val clientEventInfoBuilder = ClientEventInfoBuilder(component) val tweetItemBuilder = TweetCandidateUrtItemBuilder( clientEventInfoBuilder = clientEventInfoBuilder, timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder), feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder), socialContextBuilder = Some(CommunitiesSocialContextBuilder), tweetContext = Some(homeTweetContextBuilder) ) val moduleBuilder = TimelineModuleBuilder( entryNamespace = ConversationModuleNamespace, clientEventInfoBuilder = clientEventInfoBuilder, displayTypeBuilder = StaticModuleDisplayTypeBuilder(VerticalConversation), metadataBuilder = Some(HomeConversationModuleMetadataBuilder()) ) Some( UrtMultipleModulesDecorator( urtItemCandidateDecorator = UrtItemCandidateDecorator(tweetItemBuilder), moduleBuilder = moduleBuilder, groupByKey = (_, _, candidateFeatures) => candidateFeatures.getOrElse(ConversationModuleFocalTweetIdFeature, None) )) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeQueryTypePredicates.scala ================================================ package com.twitter.home_mixer.functional_component.decorator import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.product_mixer.core.feature.featuremap.FeatureMap object HomeQueryTypePredicates { private[this] val QueryPredicates: Seq[(String, FeatureMap => Boolean)] = Seq( ("request", _ => true), ("get_initial", _.getOrElse(GetInitialFeature, false)), ("get_newer", _.getOrElse(GetNewerFeature, false)), ("get_older", _.getOrElse(GetOlderFeature, false)), ("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)), ("request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)), ("request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)) ) val PredicateMap = QueryPredicates.toMap } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/KeywordTrendsModuleCandidateDecorator.scala ================================================ package com.twitter.home_mixer.functional_component.decorator import com.twitter.home_mixer.functional_component.decorator.KeywordTrendsModuleCandidateDecorator.Component import com.twitter.home_mixer.functional_component.decorator.KeywordTrendsModuleCandidateDecorator.KeywordTrendsEntryNamespace import com.twitter.home_mixer.functional_component.decorator.KeywordTrendsModuleCandidateDecorator.TrendsUrl import com.twitter.home_mixer.functional_component.decorator.builder.KeywordTrendMetaDescriptionBuilder import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.UrtItemInModuleDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.trend.TrendCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.trend.TrendPromotedMetadataBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.trend.TrendRankBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.StaticUrlBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.Str import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleHeaderBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedTrendCandidate import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ExternalUrl import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Vertical import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Singleton object KeywordTrendsModuleCandidateDecorator { val KeywordTrendsEntryNamespace = EntryNamespace("keyword-trends") private val Component = "trend" private val TrendsUrl = "" } @Singleton class KeywordTrendsModuleCandidateDecorator @Inject() ( keywordTrendMetaDescriptionBuilder: KeywordTrendMetaDescriptionBuilder, @ProductScoped stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) { private val clientEventInfoBuilder = ClientEventInfoBuilder[PipelineQuery, UnifiedTrendCandidate](Component) private val trendItemBuilder = TrendCandidateUrtItemBuilder( keywordTrendMetaDescriptionBuilder, TrendPromotedMetadataBuilder, clientEventInfoBuilder, Some(TrendRankBuilder[PipelineQuery, UnifiedTrendCandidate]()) ) private val trendItemDecorator = UrtItemCandidateDecorator(trendItemBuilder) private val moduleHeaderBuilder = ModuleHeaderBuilder( textBuilder = Str( text = externalStrings.TrendingString, stringCenter = stringCenter ), isSticky = Some(false), urlBuilder = Some(StaticUrlBuilder(TrendsUrl, ExternalUrl)) ) private val moduleBuilder = TimelineModuleBuilder( entryNamespace = KeywordTrendsEntryNamespace, displayTypeBuilder = StaticModuleDisplayTypeBuilder(Vertical), clientEventInfoBuilder = clientEventInfoBuilder, headerBuilder = Some(moduleHeaderBuilder) ) val moduleDecorator = UrtItemInModuleDecorator(trendItemDecorator, moduleBuilder) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/PinnedTweetBroadcastCandidateDecorator.scala ================================================ package com.twitter.home_mixer.functional_component.decorator import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.decorator.urt.builder.PinnedTweetsModuleCandidateDecorator import com.twitter.home_mixer.functional_component.decorator.urt.builder.TweetCarouselModuleCandidateDecorator import com.twitter.home_mixer.functional_component.decorator.urt.builder.TweetCarouselType import com.twitter.home_mixer.param.HomeGlobalParams.EnableLandingPage import com.twitter.home_mixer.param.HomeGlobalParams.EnablePinnedTweetsCarouselParam import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.social_context.GeneralSocialContextBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.Str import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.decorator.Decoration import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ExternalUrl import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.PinGeneralContextType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Url import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stitch.Stitch import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton class PinnedTweetBroadcastCandidateDecorator @Inject() ( tweetCarouselModuleCandidateDecorator: TweetCarouselModuleCandidateDecorator, @ProductScoped stringCenterProvider: Provider[StringCenter], externalStrings: HomeMixerExternalStrings, homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) extends CandidateDecorator[PipelineQuery, TweetCandidate] { private val stringCenter = stringCenterProvider.get() override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[Decoration]] = { if (candidates.size > 1 && query.params(EnablePinnedTweetsCarouselParam)) { tweetCarouselModuleCandidateDecorator .build(TweetCarouselType.PinnedTweets).apply(query, candidates) } else { val clientEventInfoBuilder = HomeClientEventInfoBuilder[PipelineQuery, TweetCandidate]() val landingUrl = if (query.params(EnableLandingPage)) { PinnedTweetsModuleCandidateDecorator.headerLink.map(link => Url(ExternalUrl, link)) } else None val socialContextBuilder = GeneralSocialContextBuilder( textBuilder = Str(externalStrings.BroadcastedPinnedTweetSocialContextString, stringCenter), contextType = PinGeneralContextType, landingUrl = landingUrl ) val tweetItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( clientEventInfoBuilder = clientEventInfoBuilder, socialContextBuilder = Some(socialContextBuilder), timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder), feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) ) UrtItemCandidateDecorator(tweetItemBuilder).apply(query, candidates) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/StoriesModuleCandidateDecorator.scala ================================================ package com.twitter.home_mixer.functional_component.decorator import com.twitter.home_mixer.functional_component.decorator.StoriesModuleCandidateDecorator.Component import com.twitter.home_mixer.functional_component.decorator.StoriesModuleCandidateDecorator.TrendsEntryNamespace import com.twitter.home_mixer.functional_component.decorator.StoriesModuleCandidateDecorator.TrendsUrl import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.UrtItemInModuleDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.trend.AiTrendCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.StaticUrlBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.Str import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleFooterBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleHeaderBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedTrendCandidate import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.DeepLink import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Vertical import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Singleton object StoriesModuleCandidateDecorator { val TrendsEntryNamespace = EntryNamespace("stories") private val Component = "stories" private val TrendsUrl = "" } @Singleton class StoriesModuleCandidateDecorator @Inject() ( @ProductScoped stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) { private val clientEventsBuilder = ClientEventInfoBuilder[PipelineQuery, UnifiedTrendCandidate](Component) private val trendItemBuilder = AiTrendCandidateUrtItemBuilder(clientEventsBuilder) private val trendItemDecorator = UrtItemCandidateDecorator(trendItemBuilder) private val moduleHeaderBuilder = ModuleHeaderBuilder( textBuilder = Str( text = externalStrings.NewsHeaderString, stringCenter = stringCenter ), isSticky = Some(false), urlBuilder = Some(StaticUrlBuilder(TrendsUrl, DeepLink)) ) private val moduleFooterBuilder = ModuleFooterBuilder( textBuilder = Str( text = externalStrings.NewsFooterString, stringCenter = stringCenter ), urlBuilder = Some(StaticUrlBuilder(TrendsUrl, DeepLink)) ) private val moduleBuilder = TimelineModuleBuilder( entryNamespace = TrendsEntryNamespace, displayTypeBuilder = StaticModuleDisplayTypeBuilder(Vertical), clientEventInfoBuilder = clientEventsBuilder, headerBuilder = Some(moduleHeaderBuilder), footerBuilder = Some(moduleFooterBuilder) ) val moduleDecorator: UrtItemInModuleDecorator[PipelineQuery, UnifiedTrendCandidate, Nothing] = UrtItemInModuleDecorator(trendItemDecorator, moduleBuilder) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/TuneFeedModuleCandidateDecorator.scala ================================================ package com.twitter.home_mixer.functional_component.decorator import com.twitter.home_mixer.functional_component.decorator.TuneFeedModuleCandidateDecorator.EntryNamespaceString import com.twitter.home_mixer.functional_component.decorator.TuneFeedModuleCandidateDecorator.GrokTopicBaseUrl import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder import com.twitter.home_mixer.functional_component.decorator.urt.builder.TuneFeedFeedbackActionInfoBuilder import com.twitter.home_mixer.model.HomeFeatures.CurrentDisplayedGrokTopicFeature import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.UrtItemInModuleDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.icon.HorizonIconBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.StaticUrlBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.social_context.GeneralModuleSocialContextBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.ModuleStrStatic import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.StrStatic import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleFooterBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleFooterDisplayTypeBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleHeaderBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ModuleHeaderDisplayTypeBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.decorator.Decoration import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace import com.twitter.product_mixer.core.model.marshalling.response.urt.icon.FeedbackStroke import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ExternalUrl import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TextOnlyGeneralContextType import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Feedback import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.FeedbackList import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Separator import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.util.Base64UrlSafeStringEncoder import java.nio.charset.Charset import javax.inject.Inject import javax.inject.Singleton object TuneFeedModuleCandidateDecorator { val EntryNamespaceString = "tune-feed" val GrokTopicBaseUrl = "" } @Singleton class TuneFeedModuleCandidateDecorator @Inject() ( tuneFeedFeedbackActionInfoBuilder: TuneFeedFeedbackActionInfoBuilder) extends CandidateDecorator[PipelineQuery, TweetCandidate] { override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[Decoration]] = { val clientEventInfoBuilder = HomeClientEventInfoBuilder[PipelineQuery, TweetCandidate]() val tuneFeedItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( clientEventInfoBuilder = clientEventInfoBuilder, feedbackActionInfoBuilder = Some(tuneFeedFeedbackActionInfoBuilder) ) val tuneFeedItemDecorator = UrtItemCandidateDecorator(tuneFeedItemBuilder) val tuneFeedModuleBuilder = { val generalModuleSocialContextBuilder = GeneralModuleSocialContextBuilder( textBuilder = ModuleStrStatic(text = "Tune your feed"), contextType = TextOnlyGeneralContextType ) // Safe .get, enforced in TuneFeedModuleGate val topicId = query.features.get.getOrElse(CurrentDisplayedGrokTopicFeature, None).get._1 val topicUrl = GrokTopicBaseUrl val tuneFeedModuleHeaderBuilder = ModuleHeaderBuilder( textBuilder = StrStatic(text = query.features.get .getOrElse(CurrentDisplayedGrokTopicFeature, None).map(_._2).getOrElse( "Help us show you more of what you love")), moduleSocialContextBuilder = Some(generalModuleSocialContextBuilder), moduleHeaderIconBuilder = Some(HorizonIconBuilder(FeedbackStroke)), moduleHeaderDisplayTypeBuilder = ModuleHeaderDisplayTypeBuilder(Feedback), urlBuilder = Some(StaticUrlBuilder(topicUrl, ExternalUrl)), isSticky = Some(false) ) val tuneFeedModuleFooterBuilder = ModuleFooterBuilder( textBuilder = StrStatic(text = ""), urlBuilder = None, moduleFooterDisplayTypeBuilder = ModuleFooterDisplayTypeBuilder(Separator) ) TimelineModuleBuilder( entryNamespace = EntryNamespace(EntryNamespaceString), clientEventInfoBuilder = clientEventInfoBuilder, displayTypeBuilder = StaticModuleDisplayTypeBuilder(FeedbackList), headerBuilder = Some(tuneFeedModuleHeaderBuilder), footerBuilder = Some(tuneFeedModuleFooterBuilder) ) } UrtItemInModuleDecorator(tuneFeedItemDecorator, tuneFeedModuleBuilder) .apply(query, candidates) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/TweetCarouselModuleCandidateDecorator.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder import com.twitter.home_mixer.functional_component.decorator.urt.builder.TweetCarouselType.BookmarkedTweets import com.twitter.home_mixer.functional_component.decorator.urt.builder.TweetCarouselType.CarouselType import com.twitter.home_mixer.functional_component.decorator.urt.builder.TweetCarouselType.PinnedTweets import com.twitter.home_mixer.param.HomeGlobalParams.EnableLandingPage import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.UrtItemInModuleDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.Str import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleHeaderBuilder import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace import com.twitter.product_mixer.core.model.marshalling.response.urt.icon.Frown import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.CondensedTweet import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Carousel import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Classic import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleHeader import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.core.ExternalString import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton trait ClientComponents { val clientEventComponent: String val entryNamespaceString: String val headerLink: Option[String] = None } object BookmarksModuleCandidateDecorator extends ClientComponents { override val clientEventComponent = "bookmarked-tweet" override val entryNamespaceString = "bookmarked-tweet" override val headerLink = Some("") } object PinnedTweetsModuleCandidateDecorator extends ClientComponents { override val clientEventComponent = "pinned-tweets" override val entryNamespaceString = "pinned-tweets" override val headerLink = Some("") } object TweetCarouselType extends Enumeration { type CarouselType = Value val BookmarkedTweets, PinnedTweets = Value } @Singleton class TweetCarouselModuleCandidateDecorator @Inject() ( homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, @ProductScoped stringCenterProvider: Provider[StringCenter], externalStrings: HomeMixerExternalStrings, feedbackStrings: FeedbackStrings) { private val stringCenter = stringCenterProvider.get() private def getClientComponents(carouselType: CarouselType): ClientComponents = { carouselType match { case BookmarkedTweets => BookmarksModuleCandidateDecorator case PinnedTweets => PinnedTweetsModuleCandidateDecorator } } def build( carouselType: CarouselType ): UrtItemInModuleDecorator[PipelineQuery, TweetCandidate, Nothing] = { val components = getClientComponents(carouselType) val clientEventInfoBuilder = HomeClientEventInfoBuilder[PipelineQuery, TweetCandidate]() val tweetItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( clientEventInfoBuilder = clientEventInfoBuilder, displayType = CondensedTweet, timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder), feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) ) val tweetItemDecorator = UrtItemCandidateDecorator(tweetItemBuilder) val headerBuilder = TweetCarouselModuleHeaderBuilder(carouselType, components, stringCenter, externalStrings) val tweetCarouselModuleBuilder = TimelineModuleBuilder( entryNamespace = EntryNamespace(components.entryNamespaceString), clientEventInfoBuilder = clientEventInfoBuilder, displayTypeBuilder = StaticModuleDisplayTypeBuilder(Carousel), headerBuilder = Some(headerBuilder), footerBuilder = None, feedbackActionInfoBuilder = Some( FeedbackActionInfoBuilder( seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString, seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenFeedbackString, stringCenter = stringCenter, encodedFeedbackRequest = None ) ), showMoreBehaviorBuilder = None ) UrtItemInModuleDecorator(tweetItemDecorator, tweetCarouselModuleBuilder) } } case class TweetCarouselModuleHeaderBuilder( carouselType: CarouselType, components: ClientComponents, stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) extends BaseModuleHeaderBuilder[PipelineQuery, TweetCandidate] { private def getLandingUrl(query: PipelineQuery): Option[Url] = { val landingPageUrl = components.headerLink.map { url => Url(ExternalUrl, url) } carouselType match { case BookmarkedTweets => landingPageUrl case PinnedTweets if query.params(EnableLandingPage) => landingPageUrl case _ => None } } def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Option[ModuleHeader] = { candidates.headOption.map { candidate => val title = carouselType match { case BookmarkedTweets => Str(externalStrings.BookmarksHeaderString, stringCenter) .apply(query, candidate.candidate, candidate.features) case PinnedTweets => Str(externalStrings.PinnedTweetsHeaderString, stringCenter) .apply(query, candidate.candidate, candidate.features) } val landingUrl = getLandingUrl(query) ModuleHeader( text = title, sticky = Some(false), customIcon = None, socialContext = None, icon = None, moduleHeaderDisplayType = Classic, landingUrl = landingUrl ) } } } case class FeedbackActionInfoBuilder( seeLessOftenFeedbackString: ExternalString, seeLessOftenConfirmationFeedbackString: ExternalString, stringCenter: StringCenter, encodedFeedbackRequest: Option[String]) extends BaseFeedbackActionInfoBuilder[PipelineQuery, TweetCandidate] { override def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[FeedbackActionInfo] = Some( FeedbackActionInfo( feedbackActions = Seq( FeedbackAction( feedbackType = SeeFewer, prompt = Some( Str(seeLessOftenFeedbackString, stringCenter, None) .apply(query, candidate, candidateFeatures)), confirmation = Some( Str(seeLessOftenConfirmationFeedbackString, stringCenter, None) .apply(query, candidate, candidateFeatures)), childFeedbackActions = None, feedbackUrl = None, confirmationDisplayType = None, clientEventInfo = None, richBehavior = None, subprompt = None, icon = Some(Frown), hasUndoAction = Some(true), encodedFeedbackRequest = encodedFeedbackRequest ) ), feedbackMetadata = None, displayContext = None, clientEventInfo = None ) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/VideoCarouselModuleCandidateDecorator.scala ================================================ package com.twitter.home_mixer.functional_component.decorator import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder import com.twitter.home_mixer.functional_component.decorator.urt.builder.FeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.decorator.urt.builder.FeedbackStrings import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.model.HomeFeatures.VideoDisplayTypeFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.decorator.urt.UrtItemInModuleDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.Str import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.FeatureModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.decorator.Decoration import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleFooterBuilder import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleHeaderBuilder import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.Media import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.MediaShort import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Carousel import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Classic import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.CompactCarousel import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.MediaHighCarousel import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleFooter import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleHeader import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stitch.Stitch import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.configapi.Param import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton object VideoCarouselModuleCandidateDecorator { val entryNamespaceString = "video-carousel" val deepLink = "https://twitter.com/i/video" } @Singleton class VideoCarouselModuleCandidateDecorator @Inject() ( homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, @ProductScoped stringCenterProvider: Provider[StringCenter], externalStrings: HomeMixerExternalStrings, feedbackStrings: FeedbackStrings) { import VideoCarouselModuleCandidateDecorator._ private val stringCenter = stringCenterProvider.get() def build( enableVideoCarouselFooter: Param[Boolean] ): UrtItemInModuleDecorator[PipelineQuery, TweetCandidate, Nothing] = { val clientEventInfoBuilder = HomeClientEventInfoBuilder[PipelineQuery, TweetCandidate]() val tweetItemDecorator = VideoItemCandidateDecorator(clientEventInfoBuilder, homeFeedbackActionInfoBuilder) val headerBuilder = VideoCarouselModuleHeaderBuilder( stringCenter, externalStrings, deepLink, enableVideoCarouselFooter) val footerBuilder = VideoCarouselModuleFooterBuilder( stringCenter, externalStrings, deepLink, enableVideoCarouselFooter) val videoCarouselModuleBuilder = TimelineModuleBuilder( entryNamespace = EntryNamespace(entryNamespaceString), clientEventInfoBuilder = clientEventInfoBuilder, displayTypeBuilder = FeatureModuleDisplayTypeBuilder(VideoDisplayTypeFeature, MediaHighCarousel), headerBuilder = Some(headerBuilder), footerBuilder = Some(footerBuilder), feedbackActionInfoBuilder = Some( FeedbackActionInfoBuilder( seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString, seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenFeedbackString, stringCenter = stringCenter, encodedFeedbackRequest = None ) ), showMoreBehaviorBuilder = None ) UrtItemInModuleDecorator(tweetItemDecorator, videoCarouselModuleBuilder) } } case class VideoItemCandidateDecorator( clientEventInfoBuilder: HomeClientEventInfoBuilder[PipelineQuery, TweetCandidate], homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) extends CandidateDecorator[PipelineQuery, TweetCandidate] { private val MediaVideoItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( clientEventInfoBuilder = clientEventInfoBuilder, displayType = Media, feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) ) private val MediaShortVideoItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( clientEventInfoBuilder = clientEventInfoBuilder, displayType = MediaShort, feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) ) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[Decoration]] = { val moduleDisplayType = candidates .flatMap { candidate => candidate.features.getOrElse(VideoDisplayTypeFeature, None) }.headOption.getOrElse(Carousel) // If display type of first video is Carousel, we need a horizontal video module // If display type of first video is CompactCarousel, we need a vertical video module val builder = moduleDisplayType match { case CompactCarousel => MediaShortVideoItemBuilder case _ => MediaVideoItemBuilder } val candidatePresentations = candidates.map { candidate => val itemPresentation = UrtItemPresentation( timelineItem = builder(query, candidate.candidate, candidate.features) ) Decoration(candidate.candidate, itemPresentation) } Stitch.value(candidatePresentations) } } case class VideoCarouselModuleHeaderBuilder( stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings, deepLink: String, enableVideoCarouselFooter: Param[Boolean]) extends BaseModuleHeaderBuilder[PipelineQuery, TweetCandidate] { def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Option[ModuleHeader] = { candidates.headOption.map { candidate => val title = Str(externalStrings.VideoCarouselHeaderString, stringCenter) .apply(query, candidate.candidate, candidate.features) // If footer is enabled, do not add landing URL so footer can have the deep link val landingUrl = if (!query.params(enableVideoCarouselFooter)) { Some(Url(ExternalUrl, deepLink)) } else { None } ModuleHeader( text = title, sticky = Some(false), customIcon = None, socialContext = None, icon = None, moduleHeaderDisplayType = Classic, landingUrl = landingUrl ) } } } case class VideoCarouselModuleFooterBuilder( stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings, deepLink: String, enableVideoCarouselFooter: Param[Boolean]) extends BaseModuleFooterBuilder[PipelineQuery, TweetCandidate] { def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Option[ModuleFooter] = { candidates.headOption.flatMap { candidate => // Add footer only if footer is enabled if (query.params(enableVideoCarouselFooter)) { val footerText = Str(externalStrings.VideoCarouselFooterString, stringCenter) .apply(query, candidate.candidate, candidate.features) Some( ModuleFooter( text = footerText, landingUrl = Some(Url(ExternalUrl, deepLink)) )) } else { None } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/location", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/trends_events", "src/scala/com/twitter/suggests/controller_data", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", "trends/trending_content/src/main/scala/com/twitter/trends/trending_content/util:compacting-number-localizer", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.builder import com.twitter.finagle.tracing.Trace import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TimelinesDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.suggests.controller_data.home_tweets.v1.{thriftscala => v1ht} import com.twitter.suggests.controller_data.home_tweets.{thriftscala => ht} import com.twitter.suggests.controller_data.thriftscala.ControllerData import com.twitter.suggests.controller_data.v2.thriftscala.{ControllerData => ControllerDataV2} case class HomeAdsClientEventDetailsBuilder(injectionType: Option[String]) extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] { override def apply( query: PipelineQuery, candidate: UniversalNoun[Any], candidateFeatures: FeatureMap ): Option[ClientEventDetails] = { val homeTweetsControllerDataV1 = v1ht.HomeTweetsControllerData( tweetTypesBitmap = 0L, traceId = Some(Trace.id.traceId.toLong), requestJoinId = None) val serializedControllerData = HomeClientEventDetailsBuilder.ControllerDataSerializer( ControllerData.V2( ControllerDataV2.HomeTweets(ht.HomeTweetsControllerData.V1(homeTweetsControllerDataV1)))) val clientEventDetails = ClientEventDetails( aiTrendDetails = None, conversationDetails = None, timelinesDetails = Some( TimelinesDetails( injectionType = injectionType, controllerData = Some(serializedControllerData), sourceData = None)), articleDetails = None, liveEventDetails = None, commerceDetails = None, ) Some(clientEventDetails) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.builder import com.twitter.bijection.Base64String import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.bijection.{Injection => Serializer} import com.twitter.finagle.tracing.Trace import com.twitter.home_mixer.model.HomeFeatures.PositionFeature import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TimelinesDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.suggests.controller_data.Home import com.twitter.suggests.controller_data.TweetTypeGenerator import com.twitter.suggests.controller_data.home_tweets.v1.{thriftscala => v1ht} import com.twitter.suggests.controller_data.home_tweets.{thriftscala => ht} import com.twitter.suggests.controller_data.thriftscala.ControllerData import com.twitter.suggests.controller_data.v2.thriftscala.{ControllerData => ControllerDataV2} object HomeClientEventDetailsBuilder { implicit val ByteSerializer: Serializer[ControllerData, Array[Byte]] = BinaryScalaCodec(ControllerData) val ControllerDataSerializer: Serializer[ControllerData, String] = Serializer.connect[ControllerData, Array[Byte], Base64String, String] /** * RequestJoinId field in HomeTweetsControllerData is repurposed to pass PredictionRequestId * ReqeustJoinId is no longer used. If wish to switch back, uncomment the below method and * update homeTweetsControllerDataV1.requestJoinId * * define getRequestJoinId as a method(def) rather than a val because each new request * needs to call the context to update the id. * private def getRequestJoinId(): Option[Long] = * RequestJoinKeyContext.current.flatMap(_.requestJoinId) **/ } case class HomeClientEventDetailsBuilder[-Query <: PipelineQuery, -Candidate <: UniversalNoun[Any]]( ) extends BaseClientEventDetailsBuilder[Query, Candidate] with TweetTypeGenerator[FeatureMap] { import HomeClientEventDetailsBuilder._ override def apply( query: Query, candidate: Candidate, candidateFeatures: FeatureMap ): Option[ClientEventDetails] = { val tweetTypesBitmaps = mkTweetTypesBitmaps( Home.TweetTypeIdxMap, HomeTweetTypePredicates.PredicateMap, candidateFeatures) val tweetTypesListBytes = mkItemTypesBitmapsV2( Home.TweetTypeIdxMap, HomeTweetTypePredicates.PredicateMap, candidateFeatures) val homeTweetsControllerDataV1 = v1ht.HomeTweetsControllerData( tweetTypesBitmap = tweetTypesBitmaps.getOrElse(0, 0L), tweetTypesBitmapContinued1 = tweetTypesBitmaps.get(1), traceId = Some(Trace.id.traceId.toLong), injectedPosition = candidateFeatures.getOrElse(PositionFeature, None), tweetTypesListBytes = Some(tweetTypesListBytes), // Repurpose requestJoinId field to avoid adding additional payload // Use RequestJoinId to pass PredictionRequestId for model training data join requestJoinId = candidateFeatures.getOrElse(PredictionRequestIdFeature, None), servedId = candidateFeatures.getOrElse(ServedIdFeature, None) ) val serializedControllerData = ControllerDataSerializer( ControllerData.V2( ControllerDataV2.HomeTweets(ht.HomeTweetsControllerData.V1(homeTweetsControllerDataV1)))) val clientEventDetails = ClientEventDetails( conversationDetails = None, timelinesDetails = Some( TimelinesDetails( injectionType = Some(candidateFeatures.get(ServedTypeFeature).name), controllerData = Some(serializedControllerData), sourceData = None)), articleDetails = None, liveEventDetails = None, commerceDetails = None, aiTrendDetails = None ) Some(clientEventDetails) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventInfoBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.builder import com.twitter.home_mixer.model.HomeFeatures.EntityTokenFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo import com.twitter.product_mixer.core.pipeline.PipelineQuery /** * Sets the [[ClientEventInfo]] with the `component` field set to the Suggest Type assigned to each candidate */ case class HomeClientEventInfoBuilder[Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( detailsBuilder: Option[BaseClientEventDetailsBuilder[Query, Candidate]] = None) extends BaseClientEventInfoBuilder[Query, Candidate] { override def apply( query: Query, candidate: Candidate, candidateFeatures: FeatureMap, element: Option[String] ): Option[ClientEventInfo] = { val servedType = candidateFeatures.get(ServedTypeFeature) Some( ClientEventInfo( component = Some(servedType.originalName), element = element, details = detailsBuilder.flatMap(_.apply(query, candidate, candidateFeatures)), action = None, /** * A backend entity encoded by the Client Entities Encoding Library. * Placeholder string for now */ entityToken = candidateFeatures.getOrElse(EntityTokenFeature, None) ) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeConversationModuleMetadataBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.builder import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.timeline_module.BaseModuleMetadataBuilder import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleConversationMetadata import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleMetadata import com.twitter.product_mixer.core.pipeline.PipelineQuery case class HomeConversationModuleMetadataBuilder[ -Query <: PipelineQuery, -Candidate <: BaseTweetCandidate ]() extends BaseModuleMetadataBuilder[Query, Candidate] { override def apply( query: Query, candidates: Seq[CandidateWithFeatures[Candidate]] ): ModuleMetadata = ModuleMetadata( adsMetadata = None, conversationMetadata = Some( ModuleConversationMetadata( allTweetIds = Some((candidates.last.candidate.id +: candidates.last.features.getOrElse(AncestorsFeature, Seq.empty).map(_.tweetId)).reverse), socialContext = None, enableDeduplication = Some(true) )), gridCarouselMetadata = None, pillGroupMetadata = None ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.builder import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.item.tweet.BaseTimelinesScoreInfoBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TimelinesScoreInfo import com.twitter.product_mixer.core.pipeline.PipelineQuery object HomeTimelinesScoreInfoBuilder extends BaseTimelinesScoreInfoBuilder[PipelineQuery, TweetCandidate] { private val UndefinedTweetScore = -1.0 override def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[TimelinesScoreInfo] = { if (query.params(EnableSendScoresToClient)) { val score = candidateFeatures.getOrElse(ScoreFeature, None).getOrElse(UndefinedTweetScore) Some(TimelinesScoreInfo(score)) } else None } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.builder import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationIdFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures import com.twitter.tweetypie.{thriftscala => tpt} object HomeTweetTypePredicates { /** * The predicates defined in this file are used purely for metrics tracking purposes to * measure how often we serve posts with various attributes. */ private val CandidatePredicates: Seq[(String, FeatureMap => Boolean)] = Seq( ("with_candidate", _ => true), ("retweet", _.getOrElse(IsRetweetFeature, false)), ("reply", _.getOrElse(InReplyToTweetIdFeature, None).nonEmpty), ("image", _.getOrElse(HasImageFeature, false)), ("video", _.getOrElse(HasVideoFeature, false)), ("link", _.getOrElse(EarlybirdFeature, None).exists(_.hasVisibleLink)), ("quote", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuote.contains(true))), ("like_social_context", _.getOrElse(NonSelfFavoritedByUserIdsFeature, Seq.empty).nonEmpty), ("protected", _.getOrElse(AuthorIsProtectedFeature, false)), ( "has_exclusive_conversation_author_id", _.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty), ("is_eligible_for_connect_boost", _ => false), ("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)), ("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)), ("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)), ("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)), ("get_initial", _.getOrElse(GetInitialFeature, false)), ("get_newer", _.getOrElse(GetNewerFeature, false)), ("get_middle", _.getOrElse(GetMiddleFeature, false)), ("get_older", _.getOrElse(GetOlderFeature, false)), ("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)), ("polling", _.getOrElse(PollingFeature, false)), ("near_empty", _.getOrElse(ServedSizeFeature, None).exists(_ < 3)), ("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)), ("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)), ( "less_than_10_mins_since_lnpt", _.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 10.minutes)), ("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)), ("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)), ("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)), ( "conversation_module_has_2_displayed_tweets", _.getOrElse(ConversationModule2DisplayedTweetsFeature, false)), ("empty_request", _.getOrElse(ServedSizeFeature, None).exists(_ == 0)), ("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)), ( "served_size_between_50_and_100", _.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)), ("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)), ("is_self_thread_tweet", _.getOrElse(IsSelfThreadFeature, false)), ("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty), ("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)), ("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)), ("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)), ("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)), ( "account_age_less_than_30_minutes", _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)), ("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)), ( "directed_at_user_is_in_first_degree", _.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))), ( "has_semantic_core_annotation", _.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)), ("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)), ( "account_age_less_than_1_day", _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)), ( "account_age_less_than_7_days", _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)), ( "part_of_utt", _.getOrElse(EarlybirdFeature, None) .exists(_.semanticCoreAnnotations.exists(_.exists(annotation => annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))), ( "has_home_latest_request_past_week", _.getOrElse(FollowingLastNonPollingTimeFeature, None).exists(_.untilNow < 7.days)), ("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)), ("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)), ("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)), ("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)), ("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)), ("is_random_tweet", _ => false), ("has_random_tweet_in_response", _ => false), ("is_random_tweet_above_in_utis", _ => false), ( "has_ancestor_authored_by_viewer", candidate => candidate .getOrElse(AncestorsFeature, Seq.empty).exists(ancestor => candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)), ("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)), ( "deep_reply", candidate => candidate.getOrElse(InReplyToTweetIdFeature, None).nonEmpty && candidate .getOrElse(AncestorsFeature, Seq.empty).size > 2), ( "has_simcluster_embeddings", _.getOrElse( SimclustersTweetTopKClustersWithScoresFeature, Map.empty[String, Double]).nonEmpty), ( "tweet_age_less_than_15_seconds", _.getOrElse(TweetAgeFeature, None) .exists(_ <= 15.seconds.inMillis)), ( "less_than_1_hour_since_lnpt", _.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 1.hour)), ("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))), ( "device_language_matches_tweet_language", candidate => candidate.getOrElse(TweetLanguageFeature, None) == candidate.getOrElse(DeviceLanguageFeature, None)), ( "root_ancestor", candidate => candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate .getOrElse(InReplyToTweetIdFeature, None).isEmpty), ("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))), ("in_network", _.getOrElse(InNetworkFeature, true)), ( "has_political_annotation", _.getOrElse(EarlybirdFeature, None).exists( _.semanticCoreAnnotations.exists( _.exists(annotation => SemanticCoreFeatures.PoliticalDomains.contains(annotation.domainId) || (annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy && annotation.entityId == SemanticCoreFeatures.UttPoliticsEntityId))))), ( "is_dont_at_me_by_invitation", _.getOrElse(EarlybirdFeature, None).exists( _.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.ByInvitation]))), ( "is_dont_at_me_community", _.getOrElse(EarlybirdFeature, None) .exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))), ("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)), ( "is_followed_topic_tweet", _.getOrElse(TopicContextFunctionalityTypeFeature, None) .exists(_ == BasicTopicContextFunctionalityType)), ( "is_recommended_topic_tweet", _.getOrElse(TopicContextFunctionalityTypeFeature, None) .exists(_ == RecommendationTopicContextFunctionalityType)), ("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))), ("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))), ( "has_gte_10k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10000))), ( "has_gte_100k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))), ("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)), ("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)), ( "has_gte_10_retweets", _.getOrElse(EarlybirdFeature, None).exists(_.retweetCountV2.exists(_ >= 10))), ( "has_gte_100_retweets", _.getOrElse(EarlybirdFeature, None).exists(_.retweetCountV2.exists(_ >= 100))), ( "has_gte_1k_retweets", _.getOrElse(EarlybirdFeature, None).exists(_.retweetCountV2.exists(_ >= 1000))), ( "has_us_political_annotation", _.getOrElse(EarlybirdFeature, None) .exists(_.semanticCoreAnnotations.exists(_.exists(annotation => annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy && annotation.entityId == SemanticCoreFeatures.usPoliticalTweetEntityId && annotation.groupId == SemanticCoreFeatures.UsPoliticalTweetAnnotationGroupIds.BalancedV0)))), ( "has_toxicity_score_above_threshold", _.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))), ("is_topic_tweet", _.getOrElse(TopicIdSocialContextFeature, None).isDefined), ( "text_only", candidate => candidate.getOrElse(HasDisplayedTextFeature, false) && !(candidate.getOrElse(HasImageFeature, false) || candidate.getOrElse(HasVideoFeature, false) || candidate.getOrElse(EarlybirdFeature, None).exists(_.hasCard))), ( "image_only", candidate => candidate.getOrElse(HasImageFeature, false) && !candidate.getOrElse(HasDisplayedTextFeature, false)), ("has_1_image", _.getOrElse(NumImagesFeature, None).exists(_ == 1)), ("has_2_images", _.getOrElse(NumImagesFeature, None).exists(_ == 2)), ("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)), ("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)), ("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)), ("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)), ( "has_liked_by_social_context", candidateFeatures => candidateFeatures .getOrElse(ValidLikedByUserIdsFeature, Seq.empty).nonEmpty), ( "has_followed_by_social_context", _.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty), ( "has_topic_social_context", candidateFeatures => candidateFeatures .getOrElse(TopicIdSocialContextFeature, None) .isDefined && candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined), ("video_lte_10_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ <= 10000)), ( "video_bt_10_60_sec", _.getOrElse(VideoDurationMsFeature, None).exists(duration => duration > 10000 && duration <= 60000)), ("video_gt_60_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ > 60000)), ( "tweet_age_lte_30_minutes", _.getOrElse(TweetAgeFeature, None) .exists(_ <= 30.minutes.inMillis)), ( "tweet_age_lte_1_hour", _.getOrElse(TweetAgeFeature, None) .exists(_ <= 1.hour.inMillis)), ( "tweet_age_lte_6_hours", _.getOrElse(TweetAgeFeature, None) .exists(_ <= 6.hours.inMillis)), ( "tweet_age_lte_12_hours", _.getOrElse(TweetAgeFeature, None) .exists(_ <= 12.hours.inMillis)), ( "tweet_age_gte_24_hours", _.getOrElse(TweetAgeFeature, None) .exists(_ >= 24.hours.inMillis)), ("author_is_blue_verified", _.getOrElse(AuthorIsBlueVerifiedFeature, false)), ("author_is_gold_verified", _.getOrElse(AuthorIsGoldVerifiedFeature, false)), ("author_is_gray_verified", _.getOrElse(AuthorIsGrayVerifiedFeature, false)), ("author_is_legacy_verified", _.getOrElse(AuthorIsLegacyVerifiedFeature, false)), ("author_is_creator", _.getOrElse(AuthorIsCreatorFeature, false)), ( "viral_content_creator_in_network", candidate => candidate.getOrElse(ViralContentCreatorFeature, false) && candidate.getOrElse(InNetworkFeature, true)), ( "viral_content_creator_out_of_network", candidate => candidate.getOrElse(ViralContentCreatorFeature, false) && !candidate.getOrElse(InNetworkFeature, true)), ( "grok_content_creator_in_network", candidate => candidate.getOrElse(GrokContentCreatorFeature, false) && candidate.getOrElse(InNetworkFeature, true)), ( "grok_content_creator_out_of_network", candidate => candidate.getOrElse(GrokContentCreatorFeature, false) && !candidate.getOrElse(InNetworkFeature, true)), ( "gork_content_creator_in_network", candidate => candidate.getOrElse(GorkContentCreatorFeature, false) && candidate.getOrElse(InNetworkFeature, true)), ( "gork_content_creator_out_of_network", candidate => candidate.getOrElse(GorkContentCreatorFeature, false) && !candidate.getOrElse(InNetworkFeature, true)), ("has_location", _.getOrElse(LocationIdFeature, None).isDefined), ("article", _.getOrElse(IsArticleFeature, false)), ( "grok_category_news", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_sports", _.getOrElse(GrokTopCategoryFeature, None).contains()), ("grok_category_entertainment", _ => false), ( "grok_category_business_&_finance", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_technology", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_gaming", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_movies_&_tv", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_music", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_travel", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_food", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_fashion", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_health_&_fitness", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_anime", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_celebrity", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_cryptocurrency", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_science", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_memes", _.getOrElse(GrokTopCategoryFeature, None).contains()), ("grok_category_art", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_religion", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_shopping", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_cars", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_aviation", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_motorcycles", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_beauty", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_nature_&_outdoors", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_pets", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_relationships", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_home_&_garden", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_career", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_dance", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_education", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_podcasts", _.getOrElse(GrokTopCategoryFeature, None).contains()), ( "grok_category_streaming", _.getOrElse(GrokTopCategoryFeature, None).contains()), ("grok_is_gore", _.getOrElse(GrokIsGoreFeature, None).getOrElse(false)), ("grok_is_nsfw", _.getOrElse(GrokIsNsfwFeature, None).getOrElse(false)), ("grok_is_spam", _.getOrElse(GrokIsSpamFeature, None).getOrElse(false)), ("grok_is_violent", _.getOrElse(GrokIsViolentFeature, None).getOrElse(false)), ("grok_is_low_quality", _.getOrElse(GrokIsLowQualityFeature, None).getOrElse(false)), ("grok_is_ocr", _.getOrElse(GrokIsOcrFeature, None).getOrElse(false)), ( "grok_politics_neutral", // Purely for metrics tracking. Does not affect the recommendations. _.getOrElse(GrokPoliticalInclinationFeature, None).contains(hmt.PoliticalInclination.Neutral) ), ( "grok_politics_left", // Purely for metrics tracking. Does not affect the recommendations. _.getOrElse(GrokPoliticalInclinationFeature, None).contains(hmt.PoliticalInclination.Left) ), ( "grok_politics_right", // Purely for metrics tracking. Does not affect the recommendations. _.getOrElse(GrokPoliticalInclinationFeature, None).contains(hmt.PoliticalInclination.Right) ), ("is_slop_lte_0", _.getOrElse(SlopAuthorScoreFeature, None).exists(_ <= 0.0)), ("is_slop_lte_0_2", _.getOrElse(SlopAuthorScoreFeature, None).exists(_ <= 0.2)), ("is_slop_gt_0", _.getOrElse(SlopAuthorScoreFeature, None).exists(_ > 0.0)), ("is_slop_gt_0_2", _.getOrElse(SlopAuthorScoreFeature, None).exists(_ > 0.2)), ("is_slop_gt_0_4", _.getOrElse(SlopAuthorScoreFeature, None).exists(_ > 0.4)), ("is_slop_gt_0_6", _.getOrElse(SlopAuthorScoreFeature, None).exists(_ > 0.6)), ( "unique_author_ratio_lte_50_pct", features => { val uniqueAuthorCount = features.getOrElse(UniqueAuthorCountFeature, None).getOrElse(0) val servedSize = features.getOrElse(ServedSizeFeature, None).getOrElse(1) uniqueAuthorCount.toDouble / servedSize <= 0.5 } ), ("unique_author_lte_5", _.getOrElse(UniqueAuthorCountFeature, None).exists(_ <= 5)), ("unique_author_lte_10", _.getOrElse(UniqueAuthorCountFeature, None).exists(_ <= 10)), ("unique_author_lte_15", _.getOrElse(UniqueAuthorCountFeature, None).exists(_ <= 15)), ( "single_author_gte_25_pct", features => { val maxCount = features.getOrElse(MaxSingleAuthorCountFeature, None).getOrElse(0) val servedSize = features.getOrElse(ServedSizeFeature, None).getOrElse(0) servedSize > 0 && maxCount.toDouble / servedSize >= 0.25 }), ( "single_author_gte_50_pct", features => { val maxCount = features.getOrElse(MaxSingleAuthorCountFeature, None).getOrElse(0) val servedSize = features.getOrElse(ServedSizeFeature, None).getOrElse(0) servedSize > 0 && maxCount.toDouble / servedSize >= 0.5 }), ( "unique_category_ratio_lte_50_pct", features => { val uniqueAuthorCount = features.getOrElse(UniqueCategoryCountFeature, None).getOrElse(0) val servedSize = features.getOrElse(ServedSizeFeature, None).getOrElse(1) uniqueAuthorCount.toDouble / servedSize <= 0.5 } ), ("unique_category_lte_5", _.getOrElse(UniqueCategoryCountFeature, None).exists(_ <= 5)), ("unique_category_lte_10", _.getOrElse(UniqueCategoryCountFeature, None).exists(_ <= 10)), ("unique_category_lte_15", _.getOrElse(UniqueCategoryCountFeature, None).exists(_ <= 15)), ( "single_category_gte_25_pct", features => { val maxCount = features.getOrElse(MaxSingleCategoryCountFeature, None).getOrElse(0) val servedSize = features.getOrElse(ServedSizeFeature, None).getOrElse(0) servedSize > 0 && maxCount.toDouble / servedSize >= 0.25 }), ( "single_category_gte_50_pct", features => { val maxCount = features.getOrElse(MaxSingleCategoryCountFeature, None).getOrElse(0) val servedSize = features.getOrElse(ServedSizeFeature, None).getOrElse(0) servedSize > 0 && maxCount.toDouble / servedSize >= 0.5 }), ("is_grokslopscore_low_1", _.getOrElse(GrokSlopScoreFeature, None).contains(1L)), ("is_grokslopscore_med_2", _.getOrElse(GrokSlopScoreFeature, None).contains(2L)), ("is_grokslopscore_high_3", _.getOrElse(GrokSlopScoreFeature, None).contains(3L)), ("is_boosted", _.getOrElse(IsBoostedCandidateFeature, false)), ( "has_source_signal_tweet_favorite", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetFavorite"))), ( "has_source_signal_retweet", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("Retweet"))), ( "has_source_signal_reply", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("Reply"))), ( "has_source_signal_tweet_bookmark_v1", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetBookmarkV1"))), ( "has_source_signal_original_tweet", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("OriginalTweet"))), ( "has_source_signal_account_follow", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("AccountFollow"))), ( "has_source_signal_tweet_share_v1", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetShareV1"))), ( "has_source_signal_tweet_photo_expand", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetPhotoExpand"))), ( "has_source_signal_search_tweet_click", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("SearchTweetClick"))), ( "has_source_signal_profile_tweet_click", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("ProfileTweetClick"))), ( "has_source_signal_tweet_video_open", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetVideoOpen"))), ( "has_source_signal_video_view_90d_quality_v1_all_surfaces", _.getOrElse(SourceSignalFeature, None) .exists(_.signalType.contains("VideoView90dQualityV1AllSurfaces"))), ( "has_source_signal_video_view_90d_quality_v2", _.getOrElse(SourceSignalFeature, None) .exists(_.signalType.contains("VideoView90dQualityV2"))), ( "has_source_signal_video_view_90d_quality_v2_visibility_75", _.getOrElse(SourceSignalFeature, None) .exists(_.signalType.contains("VideoView90dQualityV2Visibility75"))), ( "has_source_signal_video_view_90d_quality_v2_visibility_100", _.getOrElse(SourceSignalFeature, None) .exists(_.signalType.contains("VideoView90dQualityV2Visibility100"))), ( "has_source_signal_video_view_90d_quality_v3", _.getOrElse(SourceSignalFeature, None) .exists(_.signalType.contains("VideoView90dQualityV3"))), ( "has_source_signal_account_block", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("AccountBlock"))), ( "has_source_signal_account_mute", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("AccountMute"))), ( "has_source_signal_tweet_report", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetReport"))), ( "has_source_signal_tweet_dont_like", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetDontLike"))), ( "has_source_signal_tweet_report_v2", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetReportV2"))), ( "has_source_signal_tweet_dont_like_v2", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("TweetDontLikeV2"))), ( "has_source_signal_notification_open_and_click_v1", _.getOrElse(SourceSignalFeature, None) .exists(_.signalType.contains("NotificationOpenAndClickV1"))), ( "has_source_signal_feedback_notrelevant", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("FeedbackNotrelevant"))), ( "has_source_signal_feedback_relevant", _.getOrElse(SourceSignalFeature, None).exists(_.signalType.contains("FeedbackRelevant"))), ( "has_source_signal_high_quality_source_tweet", _.getOrElse(SourceSignalFeature, None) .exists(_.signalType.contains("HighQualitySourceTweet"))), ( "has_source_signal_high_quality_source_user", _.getOrElse(SourceSignalFeature, None) .exists(_.signalType.contains("HighQualitySourceUser"))), ) val PredicateMap = CandidatePredicates.toMap } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/KeywordTrendMetaDescriptionBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.builder import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.decorator.urt.builder.item.trend.BaseTrendMetaDescriptionBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.Str import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendTweetCountFeature import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedTrendCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.trends.trending_content.util.CompactingNumberLocalizer import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton class KeywordTrendMetaDescriptionBuilder @Inject() ( externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter]) extends BaseTrendMetaDescriptionBuilder[PipelineQuery, UnifiedTrendCandidate] { private val stringCenter = stringCenterProvider.get() private val tweetCountStr = Str( text = externalStrings.KeywordTrendsTweetCountDescriptionString, stringCenter = stringCenter ) private val compactingNumberLocalizer = new CompactingNumberLocalizer() def apply( query: PipelineQuery, candidate: UnifiedTrendCandidate, candidateFeatures: FeatureMap ): Option[String] = { // E.g. "23.4K posts" candidateFeatures.getOrElse(TrendTweetCountFeature, None).map { tweetCount => val compactedTweetCount = compactingNumberLocalizer.localizeAndCompact( query.getLanguageCode .getOrElse("en"), tweetCount) tweetCountStr(query, candidate, candidateFeatures).format(compactedTweetCount) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.builder import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventDetailsBuilder import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TimelinesDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelineservice.suggests.{thriftscala => st} case class ListClientEventDetailsBuilder(suggestType: st.SuggestType) extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] { override def apply( query: PipelineQuery, candidate: UniversalNoun[Any], candidateFeatures: FeatureMap ): Option[ClientEventDetails] = { val clientEventDetails = ClientEventDetails( conversationDetails = None, timelinesDetails = Some( TimelinesDetails( injectionType = Some(suggestType.name), controllerData = None, sourceData = None)), articleDetails = None, liveEventDetails = None, commerceDetails = None ) Some(clientEventDetails) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/VerifiedPromptBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.builder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.message.InlinePromptCandidateUrtItemStringCenterBuilder.InlinePromptClientEventInfoElement import com.twitter.product_mixer.component_library.decorator.urt.builder.item.message.MessageTextActionBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.Str import com.twitter.product_mixer.component_library.model.candidate.InlinePromptCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseClientEventInfoBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.InlinePromptMessageContent import com.twitter.product_mixer.core.model.marshalling.response.urt.item.message.MessagePromptItem import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stringcenter.client.ExternalStringRegistry import com.twitter.stringcenter.client.StringCenter case class VerifiedPromptBuilder( clientEventInfoBuilder: BaseClientEventInfoBuilder[PipelineQuery, InlinePromptCandidate], stringCenter: StringCenter, externalStringRegistry: ExternalStringRegistry) extends CandidateUrtEntryBuilder[ PipelineQuery, InlinePromptCandidate, MessagePromptItem ] { private val VerifiedUrl = "" private val headerExternalStr = externalStringRegistry.createProdString("verified_prompt_header") private val bodyExternalStr = externalStringRegistry.createProdString("verified_prompt_body") private val buttonExternalStr = externalStringRegistry.createProdString("creator_subscriptions_teaser_button") override def apply( query: PipelineQuery, candidate: InlinePromptCandidate, candidateFeatures: FeatureMap ): MessagePromptItem = { val headerStr = Str(headerExternalStr, stringCenter).apply(query, candidate, candidateFeatures) val bodyStr = Str(bodyExternalStr, stringCenter).apply(query, candidate, candidateFeatures) val buttonStrBuilder = Str(buttonExternalStr, stringCenter) val button = MessageTextActionBuilder( textBuilder = buttonStrBuilder, dismissOnClick = false, url = Some(VerifiedUrl) ).apply(query, candidate, candidateFeatures) MessagePromptItem( id = candidate.id, sortIndex = None, // Sort indexes are automatically set in the domain marshaller phase clientEventInfo = clientEventInfoBuilder( query = query, candidate = candidate, candidateFeatures = candidateFeatures, element = Some(InlinePromptClientEventInfoElement) ), isPinned = None, content = InlinePromptMessageContent( headerText = headerStr, bodyText = Some(bodyStr), primaryButtonAction = Some(button), secondaryButtonAction = None, headerRichText = None, bodyRichText = None, socialContext = None, userFacepile = None ), impressionCallbacks = None, feedbackActionInfo = None ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AddEntriesWithReplaceAndShowAlertAndShowCoverInstructionBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AlwaysInclude import com.twitter.product_mixer.component_library.premarshaller.urt.builder.IncludeInstruction import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtInstructionBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction import com.twitter.product_mixer.core.model.marshalling.response.urt.Cover import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowAlert import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry import com.twitter.product_mixer.core.pipeline.PipelineQuery case class AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder[Query <: PipelineQuery]( override val includeInstruction: IncludeInstruction[Query] = AlwaysInclude) extends UrtInstructionBuilder[Query, AddEntriesTimelineInstruction] { override def build( query: Query, entries: Seq[TimelineEntry] ): Seq[AddEntriesTimelineInstruction] = { if (includeInstruction(query, entries)) { val entriesToAdd = entries .filterNot(_.isInstanceOf[ShowAlert]) .filterNot(_.isInstanceOf[Cover]) .filter(_.entryIdToReplace.isEmpty) if (entriesToAdd.nonEmpty) Seq(AddEntriesTimelineInstruction(entriesToAdd)) else Seq.empty } else Seq.empty } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.service.{thriftscala => t} import javax.inject.Inject import javax.inject.Singleton @Singleton case class AuthorChildFeedbackActionBuilder @Inject() ( @ProductScoped stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) { def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction] = { CandidatesUtil.getOriginalAuthorId(candidateFeatures).flatMap { authorId => FeedbackUtil.buildUserSeeFewerChildFeedbackAction( userId = authorId, namesByUserId = candidateFeatures.getOrElse(ScreenNamesFeature, Map.empty[Long, String]), promptExternalString = externalStrings.showFewerTweetsString, confirmationExternalString = externalStrings.showFewerTweetsConfirmationString, engagementType = t.FeedbackEngagementType.Tweet, stringCenter = stringCenter ) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.icon import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BottomSheet import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Singleton @Singleton case class BlockUserChildFeedbackActionBuilder @Inject() ( @ProductScoped stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) { def apply( candidateFeatures: FeatureMap ): Option[ChildFeedbackAction] = { val userIdOpt = if (candidateFeatures.getOrElse(IsRetweetFeature, false)) candidateFeatures.getOrElse(SourceUserIdFeature, None) else candidateFeatures.getOrElse(AuthorIdFeature, None) userIdOpt.flatMap { userId => val screenNamesMap = candidateFeatures.getOrElse(ScreenNamesFeature, Map.empty[Long, String]) val userScreenNameOpt = screenNamesMap.get(userId) userScreenNameOpt.map { userScreenName => val prompt = stringCenter.prepare( externalStrings.blockUserString, Map("username" -> userScreenName) ) val confirmation = stringCenter.prepare( externalStrings.blockUserConfirmationString, Map("username" -> userScreenName) ) ChildFeedbackAction( feedbackType = RichBehavior, prompt = Some(prompt), confirmation = Some(confirmation), subprompt = None, feedbackUrl = None, hasUndoAction = Some(true), confirmationDisplayType = Some(BottomSheet), clientEventInfo = Some( ClientEventInfo( component = None, element = Some("block"), details = None, action = Some("click"), entityToken = None ) ), icon = Some(icon.No), richBehavior = Some(RichFeedbackBehaviorBlockUser(userId)) ) } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ChildFeedbackActionBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.param.HomeGlobalParams.PostFeedbackPromptNegativeParam import com.twitter.home_mixer.param.HomeGlobalParams.PostFeedbackPromptNeutralParam import com.twitter.home_mixer.param.HomeGlobalParams.PostFeedbackPromptPositiveParam import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.core.ExternalString import com.twitter.timelines.common.{thriftscala => tlc} import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.model.FeedbackMetadata import com.twitter.timelineservice.{thriftscala => tlst} import javax.inject.Inject import javax.inject.Singleton trait ChildFeedbackActionBuilder { val stringCenter: StringCenter def feedbackType: tlst.FeedbackType def internalFeedbackType: FeedbackType def promptString: ExternalString def confirmationString: ExternalString def clientEventElement: String def clientEventAction: String def clientEventComponent: Option[String] = None def hasUndoAction: Boolean = true def confirmationDisplayType: Option[ConfirmationDisplayType] = None def getPrompt(query: PipelineQuery): String = { stringCenter.prepare(promptString) } def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap, ): Option[ChildFeedbackAction] = { val prompt = getPrompt(query) val confirmation = stringCenter.prepare(confirmationString) val feedbackMetadata = FeedbackMetadata( engagementType = None, entityIds = Seq(tlc.FeedbackEntity.TweetId(candidate.id)), ttl = Some(FeedbackUtil.FeedbackTtl) ) val feedbackUrl = FeedbackInfo.feedbackUrl( feedbackType = feedbackType, feedbackMetadata = feedbackMetadata, injectionType = None ) Some( ChildFeedbackAction( feedbackType = internalFeedbackType, prompt = Some(prompt), confirmation = Some(confirmation), feedbackUrl = Some(feedbackUrl), hasUndoAction = Some(hasUndoAction), confirmationDisplayType = confirmationDisplayType, clientEventInfo = Some( ClientEventInfo( component = clientEventComponent, element = Some(clientEventElement), details = None, action = Some(clientEventAction), entityToken = None ) ), icon = None, richBehavior = None, subprompt = None ) ) } } @Singleton case class SeeMoreChildFeedbackActionBuilder @Inject() ( @ProductScoped override val stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) extends ChildFeedbackActionBuilder { override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Relevant override def promptString: ExternalString = externalStrings.seeMoreString override def confirmationString: ExternalString = externalStrings.genericConfirmationString override def clientEventComponent: Option[String] = Some("for_you_post_followup") override def clientEventElement: String = "feedback_relevant" override def clientEventAction: String = "click" override def internalFeedbackType: FeedbackType = Relevant override def hasUndoAction: Boolean = false override def confirmationDisplayType: Option[ConfirmationDisplayType] = Some(BottomSheet) } @Singleton case class SeeLessChildFeedbackActionBuilder @Inject() ( @ProductScoped override val stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) extends ChildFeedbackActionBuilder { override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.NotRelevant override def promptString: ExternalString = externalStrings.seeLessString override def confirmationString: ExternalString = externalStrings.genericConfirmationString override def clientEventComponent: Option[String] = Some("for_you_post_followup") override def clientEventElement: String = "feedback_notrelevant" override def clientEventAction: String = "click" override def internalFeedbackType: FeedbackType = NotRelevant override def hasUndoAction: Boolean = false override def confirmationDisplayType: Option[ConfirmationDisplayType] = Some(BottomSheet) } @Singleton case class RelevantChildFeedbackActionBuilder @Inject() ( @ProductScoped override val stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) extends ChildFeedbackActionBuilder { override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Relevant override def promptString: ExternalString = externalStrings.relevantString override def confirmationString: ExternalString = externalStrings.relevantConfirmationString override def clientEventComponent: Option[String] = Some("for_you_post_relevance_prompt") override def clientEventElement: String = "feedback_relevant" override def clientEventAction: String = "click" override def internalFeedbackType: FeedbackType = Relevant override def hasUndoAction: Boolean = false override def confirmationDisplayType: Option[ConfirmationDisplayType] = Some(BottomSheet) override def getPrompt(query: PipelineQuery): String = { query.params(PostFeedbackPromptPositiveParam) } } @Singleton case class NotRelevantChildFeedbackActionBuilder @Inject() ( @ProductScoped override val stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) extends ChildFeedbackActionBuilder { override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.NotRelevant override def promptString: ExternalString = externalStrings.notRelevantString override def confirmationString: ExternalString = externalStrings.notRelevantConfirmationString override def clientEventComponent: Option[String] = Some("for_you_post_relevance_prompt") override def clientEventElement: String = "feedback_notrelevant" override def clientEventAction: String = "click" override def internalFeedbackType: FeedbackType = NotRelevant override def hasUndoAction: Boolean = false override def confirmationDisplayType: Option[ConfirmationDisplayType] = Some(BottomSheet) override def getPrompt(query: PipelineQuery): String = { query.params(PostFeedbackPromptNegativeParam) } } @Singleton case class NeutralChildFeedbackActionBuilder @Inject() ( @ProductScoped override val stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) extends ChildFeedbackActionBuilder { override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Neutral override def promptString: ExternalString = externalStrings.neutralString override def confirmationString: ExternalString = externalStrings.neutralConfirmationString override def clientEventComponent: Option[String] = Some("for_you_post_relevance_prompt") override def clientEventElement: String = "feedback_neutral" override def clientEventAction: String = "click" override def internalFeedbackType: FeedbackType = Neutral override def hasUndoAction: Boolean = false override def confirmationDisplayType: Option[ConfirmationDisplayType] = Some(BottomSheet) override def getPrompt(query: PipelineQuery): String = { query.params(PostFeedbackPromptNeutralParam) } } @Singleton case class DontlikeNotRelevantChildFeedbackActionBuilder @Inject() ( @ProductScoped override val stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) extends ChildFeedbackActionBuilder { override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.NotRelevant override def promptString: ExternalString = externalStrings.notRelevantString override def confirmationString: ExternalString = externalStrings.notRelevantConfirmationString override def clientEventElement: String = "feedback_notrelevant" override def clientEventAction: String = "click" override def internalFeedbackType: FeedbackType = NotRelevant } @Singleton case class HatefulChildFeedbackActionBuilder @Inject() ( @ProductScoped override val stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) extends ChildFeedbackActionBuilder { override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Hateful override def promptString: ExternalString = externalStrings.hatefulString override def confirmationString: ExternalString = externalStrings.hatefulConfirmationString override def clientEventElement: String = "feedback_hateful" override def clientEventAction: String = "click" override def internalFeedbackType: FeedbackType = Hateful } @Singleton case class BoringChildFeedbackActionBuilder @Inject() ( @ProductScoped override val stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) extends ChildFeedbackActionBuilder { override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Boring override def promptString: ExternalString = externalStrings.boringString override def confirmationString: ExternalString = externalStrings.boringConfirmationString override def clientEventElement: String = "feedback_boring" override def clientEventAction: String = "click" override def internalFeedbackType: FeedbackType = Boring } @Singleton case class ConfusingChildFeedbackActionBuilder @Inject() ( @ProductScoped override val stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) extends ChildFeedbackActionBuilder { override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Confusing override def promptString: ExternalString = externalStrings.confusingString override def confirmationString: ExternalString = externalStrings.confusingConfirmationString override def clientEventElement: String = "feedback_confusing" override def clientEventAction: String = "click" override def internalFeedbackType: FeedbackType = Confusing } @Singleton case class ClickbaitChildFeedbackActionBuilder @Inject() ( @ProductScoped override val stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) extends ChildFeedbackActionBuilder { override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Clickbait override def promptString: ExternalString = externalStrings.clickbaitString override def confirmationString: ExternalString = externalStrings.clickbaitConfirmationString override def clientEventElement: String = "feedback_clickbait" override def clientEventAction: String = "click" override def internalFeedbackType: FeedbackType = Clickbait } @Singleton case class RagebaitChildFeedbackActionBuilder @Inject() ( @ProductScoped override val stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) extends ChildFeedbackActionBuilder { override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Ragebait override def promptString: ExternalString = externalStrings.ragebaitString override def confirmationString: ExternalString = externalStrings.ragebaitConfirmationString override def clientEventElement: String = "feedback_ragebait" override def clientEventAction: String = "click" override def internalFeedbackType: FeedbackType = Ragebait } @Singleton case class RegretChildFeedbackActionBuilder @Inject() ( @ProductScoped override val stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) extends ChildFeedbackActionBuilder { override def feedbackType: tlst.FeedbackType = tlst.FeedbackType.Regret override def promptString: ExternalString = externalStrings.regretString override def confirmationString: ExternalString = externalStrings.regretConfirmationString override def clientEventElement: String = "feedback_regret" override def clientEventAction: String = "click" override def internalFeedbackType: FeedbackType = Regret } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DebugSocialContextBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableDebugString import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.CommunityGeneralContextType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.DeepLink import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContext import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Url import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.util.Try object DebugSocialContextBuilder extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { val TweetUrl = "" val UserUrl = "" val TrendsUrl = "" val UserSignals = Set("Follow", "Profile") val Trends = "Trends" def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[SocialContext] = { if (query.params(EnableDebugString)) { candidateFeatures.getOrElse(DebugStringFeature, None).map { debugString => val signalId = Try(debugString.split(" ").head.toLong).toOption val baseUrl = if (UserSignals.exists(debugString.contains)) UserUrl else if (debugString.contains(Trends)) TrendsUrl else TweetUrl val url = signalId.map { id => Url( urlType = DeepLink, url = s"$baseUrl$id", urtEndpointOptions = None ) } GeneralContext( contextType = CommunityGeneralContextType, text = debugString, url = None, contextImageUrls = None, landingUrl = url ) } } else None } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdditionalChildFeedbackParam import com.twitter.home_mixer.param.HomeGlobalParams.EnableBlockMuteReportChildFeedbackParam import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.icon.Frown import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.DontLike import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackAction import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.common.{thriftscala => tlc} import com.twitter.timelineservice.model.FeedbackMetadata import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.{thriftscala => tls} import javax.inject.Inject import javax.inject.Singleton @Singleton case class DontLikeFeedbackActionBuilder @Inject() ( @ProductScoped stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings, authorChildFeedbackActionBuilder: AuthorChildFeedbackActionBuilder, retweeterChildFeedbackActionBuilder: RetweeterChildFeedbackActionBuilder, notRelevantChildFeedbackActionBuilder: DontlikeNotRelevantChildFeedbackActionBuilder, hatefulChildFeedbackActionBuilder: HatefulChildFeedbackActionBuilder, boringChildFeedbackActionBuilder: BoringChildFeedbackActionBuilder, confusingChildFeedbackActionBuilder: ConfusingChildFeedbackActionBuilder, clickbaitChildFeedbackActionBuilder: ClickbaitChildFeedbackActionBuilder, ragebaitChildFeedbackActionBuilder: RagebaitChildFeedbackActionBuilder, regretChildFeedbackActionBuilder: RegretChildFeedbackActionBuilder, blockUserChildFeedbackActionBuilder: BlockUserChildFeedbackActionBuilder, muteUserChildFeedbackActionBuilder: MuteUserChildFeedbackActionBuilder) { private val DontLikeClientEventInfo = ClientEventInfo(None, Some("feedback_dontlike"), None, Some("click"), None) def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[FeedbackAction] = { CandidatesUtil.getOriginalAuthorId(candidateFeatures).map { authorId => val feedbackEntities = Seq( tlc.FeedbackEntity.TweetId(candidate.id), tlc.FeedbackEntity.UserId(authorId) ) val feedbackMetadata = FeedbackMetadata( engagementType = None, entityIds = feedbackEntities, ttl = Some(30.days) ) val feedbackUrl = FeedbackInfo.feedbackUrl( feedbackType = tls.FeedbackType.DontLike, feedbackMetadata = feedbackMetadata, injectionType = None ) val additionalChildFeedbackActions: Seq[ChildFeedbackAction] = if (query.params(EnableAdditionalChildFeedbackParam)) { Seq( hatefulChildFeedbackActionBuilder(query, candidate, candidateFeatures), boringChildFeedbackActionBuilder(query, candidate, candidateFeatures), confusingChildFeedbackActionBuilder(query, candidate, candidateFeatures), clickbaitChildFeedbackActionBuilder(query, candidate, candidateFeatures), ragebaitChildFeedbackActionBuilder(query, candidate, candidateFeatures), regretChildFeedbackActionBuilder(query, candidate, candidateFeatures) ).flatten } else Seq.empty val blockMuteReportChildFeedbackActions: Seq[ChildFeedbackAction] = if (query.params(EnableBlockMuteReportChildFeedbackParam)) { Seq( blockUserChildFeedbackActionBuilder(candidateFeatures), muteUserChildFeedbackActionBuilder(candidateFeatures) ).flatten } else Seq.empty val childFeedbackActions = Seq( authorChildFeedbackActionBuilder(candidateFeatures), retweeterChildFeedbackActionBuilder(candidateFeatures), notRelevantChildFeedbackActionBuilder(query, candidate, candidateFeatures), ).flatten ++ additionalChildFeedbackActions ++ blockMuteReportChildFeedbackActions FeedbackAction( feedbackType = DontLike, prompt = Some(stringCenter.prepare(externalStrings.dontLikeString)), confirmation = Some(stringCenter.prepare(externalStrings.dontLikeConfirmationString)), childFeedbackActions = if (childFeedbackActions.nonEmpty) Some(childFeedbackActions) else None, feedbackUrl = Some(feedbackUrl), hasUndoAction = Some(true), confirmationDisplayType = None, clientEventInfo = Some(DontLikeClientEventInfo), icon = Some(Frown), richBehavior = None, subprompt = None, encodedFeedbackRequest = None ) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.core.ExternalString private[decorator] case class SocialContextIdAndScreenName( socialContextId: Long, screenName: String) object EngagerSocialContextBuilder { private val UserIdRequestParamName = "user_id" private val DirectInjectionContentSourceRequestParamName = "dis" private val DirectInjectionIdRequestParamName = "diid" private val DirectInjectionContentSourceSocialProofUsers = "socialproofusers" private val SocialProofUrl = "/2/timeline/social_proof.json" } case class EngagerSocialContextBuilder( contextType: GeneralContextType, stringCenter: StringCenter, oneUserString: ExternalString, twoUsersString: ExternalString, moreUsersString: ExternalString, timelineTitle: ExternalString) { import EngagerSocialContextBuilder._ def apply( socialContextIds: Seq[Long], query: PipelineQuery, candidateFeatures: FeatureMap ): Option[SocialContext] = { val realNames = candidateFeatures.getOrElse(RealNamesFeature, Map.empty[Long, String]) val validSocialContextIdAndScreenNames = socialContextIds.flatMap { socialContextId => realNames .get(socialContextId) .map(screenName => SocialContextIdAndScreenName(socialContextId, screenName)) } validSocialContextIdAndScreenNames match { case Seq(user) => val socialContextString = stringCenter.prepare(oneUserString, Map("user" -> user.screenName)) Some(mkOneUserSocialContext(socialContextString, user.socialContextId)) case Seq(firstUser, secondUser) => val socialContextString = stringCenter.prepare( twoUsersString, Map("user1" -> firstUser.screenName, "user2" -> secondUser.screenName) ) Some( mkManyUserSocialContext( socialContextString, query.getRequiredUserId, validSocialContextIdAndScreenNames.map(_.socialContextId) ) ) case firstUser +: otherUsers => val otherUsersCount = otherUsers.size val socialContextString = stringCenter.prepare( moreUsersString, Map("user" -> firstUser.screenName, "count" -> otherUsersCount) ) Some( mkManyUserSocialContext( socialContextString, query.getRequiredUserId, validSocialContextIdAndScreenNames.map(_.socialContextId) ) ) case _ => None } } private def mkOneUserSocialContext(socialContextString: String, userId: Long): GeneralContext = { GeneralContext( contextType = contextType, text = socialContextString, url = None, contextImageUrls = None, landingUrl = Some( Url( urlType = DeepLink, url = "", urtEndpointOptions = None ) ) ) } private def mkManyUserSocialContext( socialContextString: String, viewerId: Long, socialContextIds: Seq[Long] ): GeneralContext = { GeneralContext( contextType = contextType, text = socialContextString, url = None, contextImageUrls = None, landingUrl = Some( Url( urlType = UrtEndpoint, url = SocialProofUrl, urtEndpointOptions = Some(UrtEndpointOptions( requestParams = Some(Map( UserIdRequestParamName -> viewerId.toString, DirectInjectionContentSourceRequestParamName -> DirectInjectionContentSourceSocialProofUsers, DirectInjectionIdRequestParamName -> socialContextIds.mkString(",") )), title = Some(stringCenter.prepare(timelineTitle)), cacheId = None, subtitle = None )) )) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.FocalTweetRealNamesFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton /** * Use '@A replied' when the root tweet is out-of-network and the reply is in network. * * This function should only be called for the root Tweet of convo modules. This is enforced by * [[HomeTweetSocialContextBuilder]]. */ @Singleton case class ExtendedReplySocialContextBuilder @Inject() ( externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter]) extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { private val stringCenter = stringCenterProvider.get() private val extendedReplyString = externalStrings.socialContextExtendedReply def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[SocialContext] = { // If these values are missing default to not showing an extended reply banner val inNetworkRoot = candidateFeatures.getOrElse(InNetworkFeature, true) val inNetworkFocalTweet = candidateFeatures.getOrElse(FocalTweetInNetworkFeature, None).getOrElse(false) if (!inNetworkRoot && inNetworkFocalTweet) { val focalTweetAuthorIdOpt = candidateFeatures.getOrElse(FocalTweetAuthorIdFeature, None) val focalTweetRealNames = candidateFeatures .getOrElse(FocalTweetRealNamesFeature, None).getOrElse(Map.empty[Long, String]) val focalTweetAuthorNameOpt = focalTweetAuthorIdOpt.flatMap(focalTweetRealNames.get) (focalTweetAuthorIdOpt, focalTweetAuthorNameOpt) match { case (Some(focalTweetAuthorId), Some(focalTweetAuthorName)) => Some( GeneralContext( contextType = ConversationGeneralContextType, text = stringCenter .prepare(extendedReplyString, placeholders = Map("user1" -> focalTweetAuthorName)), url = None, contextImageUrls = None, landingUrl = Some( Url( urlType = DeepLink, url = "", urtEndpointOptions = None )) )) case _ => None } } else { None } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.ExternalStringRegistry import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton class FeedbackStrings @Inject() ( @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) { private val externalStringRegistry = externalStringRegistryProvider.get() val seeLessOftenFeedbackString = externalStringRegistry.createProdString("Feedback.seeLessOften") val seeLessOftenConfirmationFeedbackString = externalStringRegistry.createProdString("Feedback.seeLessOftenConfirmation") } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.conversions.DurationOps._ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SeeFewer import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.core.ExternalString import com.twitter.timelines.common.{thriftscala => tlc} import com.twitter.timelines.service.{thriftscala => t} import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.model.FeedbackMetadata import com.twitter.timelineservice.{thriftscala => tlst} object FeedbackUtil { val FeedbackTtl = 30.days def buildUserSeeFewerChildFeedbackAction( userId: Long, namesByUserId: Map[Long, String], promptExternalString: ExternalString, confirmationExternalString: ExternalString, engagementType: t.FeedbackEngagementType, stringCenter: StringCenter ): Option[ChildFeedbackAction] = { namesByUserId.get(userId).map { userScreenName => val prompt = stringCenter.prepare( promptExternalString, Map("user" -> userScreenName) ) val confirmation = stringCenter.prepare( confirmationExternalString, Map("user" -> userScreenName) ) val feedbackMetadata = FeedbackMetadata( engagementType = Some(engagementType), entityIds = Seq(tlc.FeedbackEntity.UserId(userId)), ttl = Some(FeedbackTtl)) val feedbackUrl = FeedbackInfo.feedbackUrl( feedbackType = tlst.FeedbackType.SeeFewer, feedbackMetadata = feedbackMetadata, injectionType = None ) ChildFeedbackAction( feedbackType = SeeFewer, prompt = Some(prompt), confirmation = Some(confirmation), feedbackUrl = Some(feedbackUrl), hasUndoAction = Some(true), confirmationDisplayType = None, clientEventInfo = None, icon = None, richBehavior = None, subprompt = None ) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton case class FollowedBySocialContextBuilder @Inject() ( externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter]) extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { private val stringCenter = stringCenterProvider.get() private val engagerSocialContextBuilder = EngagerSocialContextBuilder( contextType = FollowGeneralContextType, stringCenter = stringCenter, oneUserString = externalStrings.socialContextOneUserFollowsString, twoUsersString = externalStrings.socialContextTwoUsersFollowString, moreUsersString = externalStrings.socialContextMoreUsersFollowString, timelineTitle = externalStrings.socialContextFollowedByTimelineTitle ) def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[SocialContext] = { // Only apply followed-by social context for OON Tweets val inNetwork = candidateFeatures.getOrElse(InNetworkFeature, true) if (!inNetwork) { val validFollowedByUserIds = candidateFeatures.getOrElse(SGSValidFollowedByUserIdsFeature, Nil) engagerSocialContextBuilder( socialContextIds = validFollowedByUserIds, query = query, candidateFeatures = candidateFeatures ) } else { None } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.param.HomeGlobalParams.EnablePostFeedbackParam import com.twitter.home_mixer.param.HomeGlobalParams.EnablePostFollowupParam import com.twitter.home_mixer.param.HomeGlobalParams.EnablePostDetailsNegativeFeedbackParam import com.twitter.home_mixer.param.HomeGlobalParams.PostFeedbackThresholdParam import com.twitter.home_mixer.param.HomeGlobalParams.PostFollowupThresholdParam import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo import com.twitter.product_mixer.core.pipeline.PipelineQuery import javax.inject.Inject import javax.inject.Singleton import scala.util.Random @Singleton class HomeFeedbackActionInfoBuilder @Inject() ( postFeedbackActionBuilder: PostFeedbackActionBuilder, postFollowupFeedbackActionBuilder: PostFollowupFeedbackActionBuilder, dontLikeFeedbackActionBuilder: DontLikeFeedbackActionBuilder, postDetailsNegativeFeedbackActionBuilder: PostDetailsNegativeFeedbackActionBuilder) extends BaseFeedbackActionInfoBuilder[PipelineQuery, TweetCandidate] { override def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[FeedbackActionInfo] = { val supportedProduct = query.product match { case ForYouProduct => true case _ => false } val isAuthoredByViewer = CandidatesUtil.isAuthoredByViewer(query, candidateFeatures) if (supportedProduct && !isAuthoredByViewer) { // avoid showing post feedback for every candidate val shouldShowPostFeedback = query.params(EnablePostFeedbackParam) && Random.nextDouble() < query.params(PostFeedbackThresholdParam) val shouldShowPostFollowup = query.params(EnablePostFollowupParam) && Random.nextDouble() < query.params(PostFollowupThresholdParam) val shouldShowPostDetailsNegative = query.params(EnablePostDetailsNegativeFeedbackParam) val feedbackActions = Seq( if (shouldShowPostFeedback) postFeedbackActionBuilder(query, candidate, candidateFeatures) else None, if (shouldShowPostFollowup) postFollowupFeedbackActionBuilder(query, candidate, candidateFeatures) else None, dontLikeFeedbackActionBuilder( query, candidate, candidateFeatures ), if (shouldShowPostDetailsNegative) postDetailsNegativeFeedbackActionBuilder( query, candidate, candidateFeatures ) else None ).flatten Some( FeedbackActionInfo( feedbackActions = feedbackActions, feedbackMetadata = None, displayContext = None, clientEventInfo = None )) } else None } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetContextBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.BasketballContextFeature import com.twitter.home_mixer.model.HomeFeatures.GenericPostContextFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseTweetContextBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.icon import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ import com.twitter.product_mixer.core.pipeline.PipelineQuery import javax.inject.Inject import javax.inject.Singleton @Singleton class HomeTweetContextBuilder @Inject() () extends BaseTweetContextBuilder[PipelineQuery, TweetCandidate] { private val grokCom = "grok.com" private def mapBasketballStatus(originalStatus: Option[String]): Option[String] = { originalStatus.flatMap { case "Inprogress" | "Halftime" => Some("Live") case "Closed" | "Completed" => Some("Final") case "Created" | "Scheduled" => Some("Upcoming") case _ => None } } private def mapPoints(status: String, points: Option[Short]): Option[Short] = { if (status == "Live" || status == "Final") { points } else { None } } override def apply( query: PipelineQuery, candidate: TweetCandidate, features: FeatureMap ): Option[TweetContext] = { features .getOrElse(BasketballContextFeature, None).flatMap { basketballContext => val mappedStatus = mapBasketballStatus(basketballContext.status) mappedStatus.map { status => TweetContext( contextType = ContextType.Topic, text = "", landingUrl = None, contextImageUrls = None, context = Some(TweetContextDetails.Basketball(BasketballContext( clock = basketballContext.clock, homeTeamScore = mapPoints(status, basketballContext.homeTeamScore), awayTeamScore = mapPoints(status, basketballContext.awayTeamScore), homeTeamName = basketballContext.homeTeamName, awayTeamName = basketballContext.awayTeamName, status = Some(status), url = Url( urlType = DeepLink, url = basketballContext.url.url, urtEndpointOptions = None ) ))), icon = None ) } }.orElse { features.getOrElse(GenericPostContextFeature, None).map { genericContext => val grokIconOpt = if (genericContext.url.url.contains(grokCom)) Some(icon.Grok) else None TweetContext( contextType = ContextType.Topic, text = genericContext.primaryText, landingUrl = None, contextImageUrls = None, context = Some(TweetContextDetails.Generic(GenericContext( primaryText = genericContext.primaryText, secondaryText = genericContext.secondaryText, url = Url( urlType = DeepLink, url = genericContext.url.url, urtEndpointOptions = None ) ))), icon = grokIconOpt ) } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableCommunitiesContextParam import com.twitter.home_mixer.param.HomeGlobalParams.EnableSocialContextParam import com.twitter.product_mixer.component_library.decorator.urt.builder.social_context.CommunitiesSocialContextBuilder import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext import com.twitter.product_mixer.core.pipeline.PipelineQuery import javax.inject.Inject import javax.inject.Singleton @Singleton case class HomeTweetSocialContextBuilder @Inject() ( likedBySocialContextBuilder: LikedBySocialContextBuilder, servedTypeSocialContextBuilder: ServedTypeSocialContextBuilder, followedBySocialContextBuilder: FollowedBySocialContextBuilder, extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder, receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder) extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { def apply( query: PipelineQuery, candidate: TweetCandidate, features: FeatureMap ): Option[SocialContext] = { val communitiesSocialContextBuilder = if (query.params(EnableCommunitiesContextParam)) CommunitiesSocialContextBuilder(query, candidate, features) else None if (query.params(EnableSocialContextParam)) { features.getOrElse(ConversationModuleFocalTweetIdFeature, None) match { case None => DebugSocialContextBuilder(query, candidate, features) .orElse(communitiesSocialContextBuilder) .orElse(servedTypeSocialContextBuilder(query, candidate, features)) .orElse(followedBySocialContextBuilder(query, candidate, features)) case Some(_) => val conversationId = features.getOrElse(ConversationModuleIdFeature, None) // Only hydrate the social context into the root tweet in a conversation module if (conversationId.contains(candidate.id)) { communitiesSocialContextBuilder .orElse(extendedReplySocialContextBuilder(query, candidate, features)) .orElse(receivedReplySocialContextBuilder(query, candidate, features)) } else None } } else None } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.service.{thriftscala => tl} import com.twitter.timelines.util.FeedbackRequestSerializer import com.twitter.timelineservice.suggests.thriftscala.SuggestType import com.twitter.timelineservice.thriftscala.FeedbackType object HomeWhoToFollowFeedbackActionInfoBuilder { private val FeedbackMetadata = tl.FeedbackMetadata( injectionType = Some(SuggestType.WhoToFollow), engagementType = None, entityIds = Seq.empty, ttlMs = None ) private val FeedbackRequest = tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata) private val EncodedFeedbackRequest = FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest)) } @Singleton case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() ( feedbackStrings: FeedbackStrings, @ProductScoped stringCenterProvider: Provider[StringCenter]) extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] { private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder( seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString, seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString, stringCenter = stringCenterProvider.get(), encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest) ) override def apply( query: PipelineQuery, candidate: UserCandidate, candidateFeatures: FeatureMap ): Option[FeedbackActionInfo] = whoToFollowFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.service.{thriftscala => tl} import com.twitter.timelines.util.FeedbackRequestSerializer import com.twitter.timelineservice.suggests.thriftscala.SuggestType import com.twitter.timelineservice.thriftscala.FeedbackType object HomeWhoToSubscribeFeedbackActionInfoBuilder { private val FeedbackMetadata = tl.FeedbackMetadata( injectionType = Some(SuggestType.WhoToSubscribe), engagementType = None, entityIds = Seq.empty, ttlMs = None ) private val FeedbackRequest = tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata) private val EncodedFeedbackRequest = FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest)) } @Singleton case class HomeWhoToSubscribeFeedbackActionInfoBuilder @Inject() ( feedbackStrings: FeedbackStrings, @ProductScoped stringCenterProvider: Provider[StringCenter]) extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] { private val whoToSubscribeFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder( seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString, seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString, stringCenter = stringCenterProvider.get(), encodedFeedbackRequest = Some(HomeWhoToSubscribeFeedbackActionInfoBuilder.EncodedFeedbackRequest) ) override def apply( query: PipelineQuery, candidate: UserCandidate, candidateFeatures: FeatureMap ): Option[FeedbackActionInfo] = whoToSubscribeFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.ValidLikedByUserIdsFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.LikeGeneralContextType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton case class LikedBySocialContextBuilder @Inject() ( externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter]) extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { private val stringCenter = stringCenterProvider.get() private val engagerSocialContextBuilder = EngagerSocialContextBuilder( contextType = LikeGeneralContextType, stringCenter = stringCenter, oneUserString = externalStrings.socialContextOneUserLikedString, twoUsersString = externalStrings.socialContextTwoUsersLikedString, moreUsersString = externalStrings.socialContextMoreUsersLikedString, timelineTitle = externalStrings.socialContextLikedByTimelineTitle ) def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[SocialContext] = { // Liked by users are valid only if they pass both the SGS and Perspective filters. val validLikedByUserIds = candidateFeatures.getOrElse(ValidLikedByUserIdsFeature, Nil) engagerSocialContextBuilder( socialContextIds = validLikedByUserIds, query = query, candidateFeatures = candidateFeatures ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelineservice.suggests.{thriftscala => t} import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton /** * "Your Lists" will be rendered for the context and a url link for your lists. */ @Singleton case class ListsSocialContextBuilder @Inject() ( externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter]) extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { private val stringCenter = stringCenterProvider.get() private val listString = externalStrings.ownedSubscribedListsModuleHeaderString def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[SocialContext] = { candidateFeatures.get(SuggestTypeFeature) match { case Some(suggestType) if suggestType == t.SuggestType.RankedListTweet => val userName = query.features.flatMap(_.getOrElse(UserScreenNameFeature, None)) Some( GeneralContext( contextType = ListGeneralContextType, text = stringCenter.prepare(listString), url = userName.map(name => ""), contextImageUrls = None, landingUrl = None )) case _ => None } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.icon import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BottomSheet import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Singleton @Singleton case class MuteUserChildFeedbackActionBuilder @Inject() ( @ProductScoped stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) { def apply( candidateFeatures: FeatureMap ): Option[ChildFeedbackAction] = { val userIdOpt = if (candidateFeatures.getOrElse(IsRetweetFeature, false)) candidateFeatures.getOrElse(SourceUserIdFeature, None) else candidateFeatures.getOrElse(AuthorIdFeature, None) userIdOpt.flatMap { userId => val screenNamesMap = candidateFeatures.getOrElse(ScreenNamesFeature, Map.empty[Long, String]) val userScreenNameOpt = screenNamesMap.get(userId) userScreenNameOpt.map { userScreenName => val prompt = stringCenter.prepare( externalStrings.muteUserString, Map("username" -> userScreenName) ) val confirmation = stringCenter.prepare( externalStrings.muteUserConfirmationString, Map("username" -> userScreenName) ) ChildFeedbackAction( feedbackType = RichBehavior, prompt = Some(prompt), confirmation = Some(confirmation), subprompt = None, feedbackUrl = None, hasUndoAction = Some(true), confirmationDisplayType = Some(BottomSheet), clientEventInfo = Some( ClientEventInfo( component = None, element = Some("mute"), details = None, action = Some("click"), entityToken = None ) ), icon = Some(icon.SpeakerOff), richBehavior = Some(RichFeedbackBehaviorToggleMuteUser(userId)), ) } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackAction import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecWithEducationTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Singleton @Singleton case class NotInterestedTopicFeedbackActionBuilder @Inject() ( @ProductScoped stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) { def apply( candidateFeatures: FeatureMap ): Option[FeedbackAction] = { val isOutOfNetwork = !candidateFeatures.getOrElse(InNetworkFeature, true) val validFollowedByUserIds = candidateFeatures.getOrElse(SGSValidFollowedByUserIdsFeature, Nil) val validLikedByUserIds = candidateFeatures .getOrElse(SGSValidLikedByUserIdsFeature, Nil) .filter( candidateFeatures.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Nil).toSet.contains) if (isOutOfNetwork && validLikedByUserIds.isEmpty && validFollowedByUserIds.isEmpty) { val topicIdSocialContext = candidateFeatures.getOrElse(TopicIdSocialContextFeature, None) val topicContextFunctionalityType = candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None) (topicIdSocialContext, topicContextFunctionalityType) match { case (Some(topicId), Some(topicContextFunctionalityType)) if topicContextFunctionalityType == RecommendationTopicContextFunctionalityType || topicContextFunctionalityType == RecWithEducationTopicContextFunctionalityType => Some( FeedbackAction( feedbackType = RichBehavior, prompt = None, confirmation = None, childFeedbackActions = None, feedbackUrl = None, hasUndoAction = Some(true), confirmationDisplayType = None, clientEventInfo = None, icon = None, richBehavior = Some(RichFeedbackBehaviorMarkNotInterestedTopic(topicId = topicId.toString)), subprompt = None, encodedFeedbackRequest = None ) ) case _ => None } } else { None } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.NotRelevant import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.common.{thriftscala => tlc} import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.model.FeedbackMetadata import com.twitter.timelineservice.{thriftscala => tlst} import javax.inject.Inject import javax.inject.Singleton @Singleton case class NotRelevantChildFeedbackActionBuilder @Inject() ( @ProductScoped stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) { def apply( candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[ChildFeedbackAction] = { val prompt = stringCenter.prepare(externalStrings.notRelevantString) val confirmation = stringCenter.prepare(externalStrings.notRelevantConfirmationString) val feedbackMetadata = FeedbackMetadata( engagementType = None, entityIds = Seq(tlc.FeedbackEntity.TweetId(candidate.id)), ttl = Some(FeedbackUtil.FeedbackTtl)) val feedbackUrl = FeedbackInfo.feedbackUrl( feedbackType = tlst.FeedbackType.NotRelevant, feedbackMetadata = feedbackMetadata, injectionType = candidateFeatures.getOrElse(SuggestTypeFeature, None) ) Some( ChildFeedbackAction( feedbackType = NotRelevant, prompt = Some(prompt), confirmation = Some(confirmation), feedbackUrl = Some(feedbackUrl), hasUndoAction = Some(true), confirmationDisplayType = None, clientEventInfo = None, icon = None, richBehavior = None, subprompt = None ) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelineservice.suggests.{thriftscala => st} import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton case class PopularInYourAreaSocialContextBuilder @Inject() ( externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter]) extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { private val stringCenter = stringCenterProvider.get() private val popularInYourAreaString = externalStrings.socialContextPopularInYourAreaString def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[SocialContext] = { val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None) if (suggestTypeOpt.contains(st.SuggestType.RecommendedTrendTweet)) { Some( GeneralContext( contextType = LocationGeneralContextType, text = stringCenter.prepare(popularInYourAreaString), url = None, contextImageUrls = None, landingUrl = None )) } else None } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelineservice.suggests.{thriftscala => st} import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton case class PopularVideoSocialContextBuilder @Inject() ( externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter]) extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { private val stringCenter = stringCenterProvider.get() private val popularVideoString = externalStrings.socialContextPopularVideoString def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[SocialContext] = { val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None) if (suggestTypeOpt.contains(st.SuggestType.MediaTweet)) { Some( GeneralContext( contextType = SparkleGeneralContextType, text = stringCenter.prepare(popularVideoString), url = None, contextImageUrls = None, landingUrl = Some( Url( urlType = DeepLink, url = "" ) ) )) } else None } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PostDetailsNegativeFeedbackActionBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.icon.Frown import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BottomSheet import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackAction import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.NotRelevant import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.common.{thriftscala => tlc} import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.model.FeedbackMetadata import com.twitter.timelineservice.{thriftscala => tls} import javax.inject.Inject import javax.inject.Singleton @Singleton case class PostDetailsNegativeFeedbackActionBuilder @Inject() ( @ProductScoped stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) { private val PostDetailsNegativeClientEventInfo = ClientEventInfo(None, Some("feedback_notrelevant"), None, Some("click"), None) def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[FeedbackAction] = { CandidatesUtil.getOriginalAuthorId(candidateFeatures).map { authorId => val feedbackEntities = Seq( tlc.FeedbackEntity.TweetId(candidate.id), tlc.FeedbackEntity.UserId(authorId) ) val feedbackMetadata = FeedbackMetadata( engagementType = None, entityIds = feedbackEntities, ttl = Some(30.days) ) val feedbackUrl = FeedbackInfo.feedbackUrl( feedbackType = tls.FeedbackType.NotRelevant, feedbackMetadata = feedbackMetadata, injectionType = None ) FeedbackAction( feedbackType = NotRelevant, prompt = Some(stringCenter.prepare(externalStrings.notRelevantString)), confirmation = Some(stringCenter.prepare(externalStrings.notRelevantConfirmationString)), childFeedbackActions = None, feedbackUrl = Some(feedbackUrl), hasUndoAction = Some(false), confirmationDisplayType = Some(BottomSheet), clientEventInfo = Some(PostDetailsNegativeClientEventInfo), icon = Some(Frown), richBehavior = None, subprompt = None, encodedFeedbackRequest = None ) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PostFeedbackActionBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.param.HomeGlobalParams.PostFeedbackPromptTitleParam import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.icon.Smile import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BottomSheet import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackAction import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Generic import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.common.{thriftscala => tlc} import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.model.FeedbackMetadata import com.twitter.timelineservice.{thriftscala => tls} import javax.inject.Inject import javax.inject.Singleton @Singleton case class PostFeedbackActionBuilder @Inject() ( @ProductScoped stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings, relevantChildFeedbackActionBuilder: RelevantChildFeedbackActionBuilder, notRelevantChildFeedbackActionBuilder: NotRelevantChildFeedbackActionBuilder, neutralChildFeedbackActionBuilder: NeutralChildFeedbackActionBuilder) { val ClientEventInfoComponent: String = "for_you_post_relevance_prompt" val ClientEventInfoElement: String = "relevance_prompt" def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[FeedbackAction] = { CandidatesUtil.getOriginalAuthorId(candidateFeatures).map { authorId => val feedbackEntities = Seq( tlc.FeedbackEntity.TweetId(candidate.id), tlc.FeedbackEntity.UserId(authorId) ) val feedbackMetadata = FeedbackMetadata( engagementType = None, entityIds = feedbackEntities, ttl = Some(30.days) ) val feedbackUrl = FeedbackInfo.feedbackUrl( feedbackType = tls.FeedbackType.Generic, feedbackMetadata = feedbackMetadata, injectionType = None ) val childFeedbackActions: Seq[ChildFeedbackAction] = { Seq( relevantChildFeedbackActionBuilder(query, candidate, candidateFeatures), notRelevantChildFeedbackActionBuilder(query, candidate, candidateFeatures), // neutralChildFeedbackActionBuilder(query, candidate, candidateFeatures) ).flatten } FeedbackAction( feedbackType = Generic, prompt = Some(query.params(PostFeedbackPromptTitleParam)), confirmation = Some( stringCenter.prepare(externalStrings.genericConfirmationString) ), childFeedbackActions = Some(childFeedbackActions), feedbackUrl = Some(feedbackUrl), hasUndoAction = None, confirmationDisplayType = Some(BottomSheet), clientEventInfo = Some( ClientEventInfo( component = Some(ClientEventInfoComponent), element = Some(ClientEventInfoElement), details = None, action = None, entityToken = None )), icon = Some(Smile), richBehavior = None, subprompt = None, encodedFeedbackRequest = None ) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PostFollowupFeedbackActionBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.param.HomeGlobalParams.PostFeedbackPromptTitleParam import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BottomSheet import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackAction import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GiveFeedback import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.common.{thriftscala => tlc} import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.model.FeedbackMetadata import com.twitter.timelineservice.{thriftscala => tls} import javax.inject.Inject import javax.inject.Singleton @Singleton case class PostFollowupFeedbackActionBuilder @Inject() ( @ProductScoped stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings, seeMoreChildFeedbackActionBuilder: SeeMoreChildFeedbackActionBuilder, seeLessChildFeedbackActionBuilder: SeeLessChildFeedbackActionBuilder) { val ClientEventInfoComponent: String = "for_you_post_followup" val ClientEventInfoElement: String = "followup" def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[FeedbackAction] = { CandidatesUtil.getOriginalAuthorId(candidateFeatures).map { authorId => val feedbackEntities = Seq( tlc.FeedbackEntity.TweetId(candidate.id), tlc.FeedbackEntity.UserId(authorId) ) val feedbackMetadata = FeedbackMetadata( engagementType = None, entityIds = feedbackEntities, ttl = Some(30.days) ) val feedbackUrl = FeedbackInfo.feedbackUrl( feedbackType = tls.FeedbackType.Generic, feedbackMetadata = feedbackMetadata, injectionType = None ) val childFeedbackActions: Seq[ChildFeedbackAction] = { Seq( seeLessChildFeedbackActionBuilder(query, candidate, candidateFeatures), seeMoreChildFeedbackActionBuilder(query, candidate, candidateFeatures) ).flatten } FeedbackAction( feedbackType = GiveFeedback, prompt = Some(query.params(PostFeedbackPromptTitleParam)), confirmation = Some( stringCenter.prepare(externalStrings.genericConfirmationString) ), childFeedbackActions = Some(childFeedbackActions), feedbackUrl = Some(feedbackUrl), hasUndoAction = None, confirmationDisplayType = Some(BottomSheet), clientEventInfo = Some( ClientEventInfo( component = Some(ClientEventInfoComponent), element = Some(ClientEventInfoElement), details = None, action = None, entityToken = None )), icon = None, richBehavior = None, subprompt = None, encodedFeedbackRequest = None ) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton /** * Use '@A received a reply' as social context when the root Tweet is in network and the focal tweet is OON. * * This function should only be called for the root Tweet of convo modules. This is enforced by * [[HomeTweetSocialContextBuilder]]. */ @Singleton case class ReceivedReplySocialContextBuilder @Inject() ( externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter]) extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { private val stringCenter = stringCenterProvider.get() private val receivedReplyString = externalStrings.socialContextReceivedReply def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[SocialContext] = { // If these values are missing default to not showing a received a reply banner val inNetwork = candidateFeatures.getOrElse(InNetworkFeature, false) val inNetworkFocalTweet = candidateFeatures.getOrElse(FocalTweetInNetworkFeature, None).getOrElse(true) if (inNetwork && !inNetworkFocalTweet) { val authorIdOpt = candidateFeatures.getOrElse(AuthorIdFeature, None) val realNames = candidateFeatures.getOrElse(RealNamesFeature, Map.empty[Long, String]) val authorNameOpt = authorIdOpt.flatMap(realNames.get) (authorIdOpt, authorNameOpt) match { case (Some(authorId), Some(authorName)) => Some( GeneralContext( contextType = ConversationGeneralContextType, text = stringCenter .prepare(receivedReplyString, placeholders = Map("user1" -> authorName)), url = None, contextImageUrls = None, landingUrl = Some( Url( urlType = DeepLink, url = "", urtEndpointOptions = None ) ) ) ) case _ => None } } else { None } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RelevancePromptCandidateUrtItemBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature import com.twitter.product_mixer.component_library.decorator.urt.builder.item.relevance_prompt.RelevancePromptCandidateUrtItemStringCenterBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.stringcenter.StrStatic import com.twitter.product_mixer.component_library.model.candidate.RelevancePromptCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.CandidateUrtEntryBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.item.prompt.Compact import com.twitter.product_mixer.core.model.marshalling.response.urt.item.prompt.PromptItem import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Callback import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.common.{thriftscala => thriftCommon} import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.service.thriftscala.FeedbackEngagementType import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.model.FeedbackMetadata import com.twitter.timelineservice.suggests.thriftscala.SuggestType import com.twitter.timelineservice.{thriftscala => thrift} object RelevancePromptCandidateUrtItemBuilder { val ClientEventInfoComponent: String = "for_you_relevance_prompt" val ClientEventBuilder = ClientEventInfoBuilder[PipelineQuery, RelevancePromptCandidate]( component = ClientEventInfoComponent ) val ConfirmationStr = "Thank you for your feedback!" } case class RelevancePromptCandidateUrtItemBuilder( titleParam: FSParam[String], relevantPromptParam: FSParam[String], notRelevantPromptParam: FSParam[String], neutralPromptParam: FSParam[String], ) extends CandidateUrtEntryBuilder[ PipelineQuery, RelevancePromptCandidate, PromptItem ] { import RelevancePromptCandidateUrtItemBuilder._ override def apply( query: PipelineQuery, candidate: RelevancePromptCandidate, candidateFeatures: FeatureMap ): PromptItem = { val servedId: Long = query.features.flatMap(_.getOrElse(ServedIdFeature, None)).getOrElse(0L) // Use user 0 as a dummy entity ID until we have a proper session ID to track val feedbackMetadata = FeedbackMetadata( Some(FeedbackEngagementType.RelevancePrompt), entityIds = Seq(thriftCommon.FeedbackEntity.FeedbackId(servedId)), ttl = Some(30.days) ) val positiveFeedbackUrl = FeedbackInfo.feedbackUrl( feedbackType = thrift.FeedbackType.Relevant, feedbackMetadata = feedbackMetadata, injectionType = Some(SuggestType.RelevancePrompt) ) val negativeFeedbackUrl = FeedbackInfo.feedbackUrl( feedbackType = thrift.FeedbackType.NotRelevant, feedbackMetadata = feedbackMetadata, injectionType = Some(SuggestType.RelevancePrompt) ) val neutralFeedbackUrl = FeedbackInfo.feedbackUrl( feedbackType = thrift.FeedbackType.Neutral, feedbackMetadata = feedbackMetadata, injectionType = Some(SuggestType.RelevancePrompt) ) val relevancePromptCandidateUrtItemStringCenterBuilder = RelevancePromptCandidateUrtItemStringCenterBuilder( clientEventInfoBuilder = ClientEventBuilder, titleTextBuilder = StrStatic(query.params(titleParam)), confirmationTextBuilder = StrStatic(ConfirmationStr), isRelevantTextBuilder = StrStatic(query.params(relevantPromptParam)), notRelevantTextBuilder = StrStatic(query.params(notRelevantPromptParam)), displayType = Compact, isRelevantCallback = Callback(positiveFeedbackUrl), notRelevantCallback = Callback(negativeFeedbackUrl), neutralTextBuilder = Some(StrStatic(query.params(neutralPromptParam))), neutralCallback = Some(Callback(neutralFeedbackUrl)) ) relevancePromptCandidateUrtItemStringCenterBuilder( query = query, relevancePromptCandidate = candidate, candidateFeatures = candidateFeatures ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.marshalling.response.urt.icon import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ClientEventInfo import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Singleton @Singleton case class ReportTweetChildFeedbackActionBuilder @Inject() ( @ProductScoped stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) { def apply( candidate: TweetCandidate, ): Option[ChildFeedbackAction] = { Some( ChildFeedbackAction( feedbackType = RichBehavior, prompt = Some(stringCenter.prepare(externalStrings.reportTweetString)), confirmation = None, feedbackUrl = None, hasUndoAction = Some(true), confirmationDisplayType = None, clientEventInfo = Some( ClientEventInfo( component = None, element = Some("report_tweet"), details = None, action = Some("click"), entityToken = None ) ), icon = Some(icon.Flag), richBehavior = Some(RichFeedbackBehaviorReportTweet(candidate.id)), subprompt = None ) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.service.{thriftscala => t} import javax.inject.Inject import javax.inject.Singleton @Singleton case class RetweeterChildFeedbackActionBuilder @Inject() ( @ProductScoped stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) { def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction] = { val isRetweet = candidateFeatures.getOrElse(IsRetweetFeature, false) if (isRetweet) { candidateFeatures.getOrElse(AuthorIdFeature, None).flatMap { retweeterId => FeedbackUtil.buildUserSeeFewerChildFeedbackAction( userId = retweeterId, namesByUserId = candidateFeatures.getOrElse(ScreenNamesFeature, Map.empty[Long, String]), promptExternalString = externalStrings.showFewerRetweetsString, confirmationExternalString = externalStrings.showFewerRetweetsConfirmationString, engagementType = t.FeedbackEngagementType.Retweet, stringCenter = stringCenter ) } } else None } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ServedTypeSocialContextBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityNameFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton /** * Builds social context for tweets that have a 1:1 relationship * between served type and social context. e.g. Lists/Communities */ @Singleton case class ServedTypeSocialContextBuilder @Inject() ( externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter]) extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { private val ListsUrl = "" private val CommunitiesUrl = "" private val stringCenter = stringCenterProvider.get() private val listString = externalStrings.ownedSubscribedListsModuleHeaderString private val popularGeoString = externalStrings.socialContextPopularInYourAreaString def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[SocialContext] = candidateFeatures.get(ServedTypeFeature) match { case t.ServedType.ForYouCommunity => val communityId = candidateFeatures.getOrElse(CommunityIdFeature, None) val communityName = candidateFeatures.getOrElse(CommunityNameFeature, None) val context = GeneralContext( contextType = CommunityGeneralContextType, text = communityName.getOrElse(""), url = None, contextImageUrls = None, landingUrl = communityId.map(id => Url(ExternalUrl, CommunitiesUrl + id)) ) Some(context) case t.ServedType.ForYouPopularGeo => val context = GeneralContext( contextType = LocationGeneralContextType, text = stringCenter.prepare(popularGeoString), url = None, contextImageUrls = None, landingUrl = None ) Some(context) case _ => None } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContext import com.twitter.product_mixer.core.pipeline.PipelineQuery import javax.inject.Inject import javax.inject.Singleton @Singleton case class TopicSocialContextBuilder @Inject() () extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[SocialContext] = { val inNetwork = candidateFeatures.getOrElse(InNetworkFeature, true) if (!inNetwork) { val topicIdSocialContextOpt = candidateFeatures.getOrElse(TopicIdSocialContextFeature, None) val topicContextFunctionalityTypeOpt = candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None) (topicIdSocialContextOpt, topicContextFunctionalityTypeOpt) match { case (Some(topicId), Some(topicContextFunctionalityType)) => Some( TopicContext( topicId = topicId.toString, functionalityType = Some(topicContextFunctionalityType) )) case _ => None } } else { None } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TuneFeedFeedbackActionInfoBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo import com.twitter.product_mixer.core.pipeline.PipelineQuery import javax.inject.Inject import javax.inject.Singleton @Singleton class TuneFeedFeedbackActionInfoBuilder @Inject() ( postFollowupFeedbackActionBuilder: PostFollowupFeedbackActionBuilder, dontLikeFeedbackActionBuilder: DontLikeFeedbackActionBuilder) extends BaseFeedbackActionInfoBuilder[PipelineQuery, TweetCandidate] { override def apply( query: PipelineQuery, candidate: TweetCandidate, candidateFeatures: FeatureMap ): Option[FeedbackActionInfo] = { Some( FeedbackActionInfo( feedbackActions = Seq( postFollowupFeedbackActionBuilder(query, candidate, candidateFeatures), dontLikeFeedbackActionBuilder(query, candidate, candidateFeatures)).flatten, feedbackMetadata = None, displayContext = None, clientEventInfo = None )) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.decorator.urt.builder import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.response.urt.icon import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichBehavior import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import javax.inject.Inject import javax.inject.Singleton @Singleton case class UnfollowUserChildFeedbackActionBuilder @Inject() ( @ProductScoped stringCenter: StringCenter, externalStrings: HomeMixerExternalStrings) { def apply(candidateFeatures: FeatureMap): Option[ChildFeedbackAction] = { val isInNetwork = candidateFeatures.getOrElse(InNetworkFeature, false) val userIdOpt = candidateFeatures.getOrElse(AuthorIdFeature, None) if (isInNetwork) { userIdOpt.flatMap { userId => val screenNamesMap = candidateFeatures.getOrElse(ScreenNamesFeature, Map.empty[Long, String]) val userScreenNameOpt = screenNamesMap.get(userId) userScreenNameOpt.map { userScreenName => val prompt = stringCenter.prepare( externalStrings.unfollowUserString, Map("username" -> userScreenName) ) val confirmation = stringCenter.prepare( externalStrings.unfollowUserConfirmationString, Map("username" -> userScreenName) ) ChildFeedbackAction( feedbackType = RichBehavior, prompt = Some(prompt), confirmation = Some(confirmation), feedbackUrl = None, hasUndoAction = Some(true), confirmationDisplayType = None, clientEventInfo = None, icon = Some(icon.Unfollow), richBehavior = Some(RichFeedbackBehaviorToggleFollowUser(userId)), subprompt = None ) } } } else None } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AncestorFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} import com.twitter.tweetconvosvc.{thriftscala => tcs} import javax.inject.Inject import javax.inject.Singleton @Singleton class AncestorFeatureHydrator @Inject() ( conversationServiceClient: tcs.ConversationService.MethodPerEndpoint) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Ancestor") override val features: Set[Feature[_, _]] = Set(AncestorsFeature) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]], ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val candidatesWithReplies = candidates.collect { case candidate if candidate.features.getOrElse(InReplyToTweetIdFeature, None).isDefined => candidate.candidate.id } val candidateIsReplyIndexMap = candidatesWithReplies.zipWithIndex.toMap val ancestorsRequest = tcs.GetAncestorsRequest(candidatesWithReplies) conversationServiceClient.getAncestors(ancestorsRequest).map { getAncestorsResponse => candidates.map { candidate => val resultIndex = candidateIsReplyIndexMap.get(candidate.candidate.id) val ancestors = resultIndex .map { index => getAncestorsResponse.ancestors(index) match { case tcs.TweetAncestorsResult.TweetAncestors(ancestorsResult) if ancestorsResult.nonEmpty => ancestorsResult.head.ancestors ++ getTruncatedRootTweet(ancestorsResult.head) case _ => Seq.empty } }.getOrElse(Seq.empty) FeatureMap(AncestorsFeature, ancestors) } } } private def getTruncatedRootTweet( ancestors: ta.TweetAncestors, ): Option[ta.TweetAncestor] = { ancestors.conversationRootAuthorId.collect { case rootAuthorId if ancestors.state == ta.ReplyState.Partial && ancestors.ancestors.last.tweetId != ancestors.conversationId => ta.TweetAncestor(ancestors.conversationId, rootAuthorId) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AuthorFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features.AuthorFeaturesAdapter import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.AuthorFeatureRepository import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.repository.KeyValueRepository import com.twitter.servo.repository.KeyValueResult import com.twitter.stitch.Stitch import com.twitter.timelines.author_features.v1.{thriftjava => af} import com.twitter.util.Future import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object AuthorFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class AuthorFeatureHydrator @Inject() ( @Named(AuthorFeatureRepository) client: KeyValueRepository[Seq[Long], Long, af.AuthorFeatures], override val statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("AuthorFeature") override val features: Set[Feature[_, _]] = Set(AuthorFeature) override val statScope: String = identifier.toString private val DefaultDataRecord = new DataRecord() private val DefaultFeatureMap = FeatureMap(AuthorFeature, DefaultDataRecord) private val authorIdsCountStat = statsReceiver.scope(statScope).stat("authorIdsSize") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val inNetworkCandidates = candidates.filter(_.features.getOrElse(FromInNetworkSourceFeature, false)) val possiblyAuthorIds = inNetworkCandidates.map(extractKey) val authorIds = possiblyAuthorIds.flatten.distinct authorIdsCountStat.add(authorIds.size) val response: Future[KeyValueResult[Long, DataRecord]] = if (authorIds.nonEmpty) client(authorIds) .map { _.mapFound { AuthorFeaturesAdapter.adaptToDataRecords(_).asScala.head } } else Future.value(KeyValueResult.empty) response.map { result => candidates.map { candidate => val authorId = extractKey(candidate) val authorDR = observedGet(key = authorId, keyValueResult = result) authorDR.toOption .flatMap { _.map { features => FeatureMap(AuthorFeature, features) } }.getOrElse(DefaultFeatureMap) } } } private def extractKey( candidate: CandidateWithFeatures[TweetCandidate] ): Option[Long] = { CandidatesUtil.getOriginalAuthorId(candidate.features) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/AuthorLargeEmbeddingsFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.AuthorLargeEmbeddingsFeature import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.AuthorLargeEmbeddingsKeyFeature import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableLargeEmbeddingsFeatureHydrationParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelNameParam import com.twitter.home_mixer_features.{thriftscala => hmf} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.large_embeddings.AuthorLargeEmbeddingsAdapter import com.twitter.timelines.prediction.adapters.large_embeddings.HashingFeatureParams import com.twitter.timelines.prediction.adapters.large_embeddings.HomeMixerLargeEmbeddingsFeatureHydrator import com.twitter.timelines.prediction.adapters.large_embeddings.LargeEmbeddingsAdapter import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton @Singleton class AuthorLargeEmbeddingsFeatureHydrator @Inject() ( statsReceiver: StatsReceiver, override val homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] with HomeMixerLargeEmbeddingsFeatureHydrator { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("AuthorLargeEmbeddings") override val features: Set[Feature[_, _]] = Set(AuthorLargeEmbeddingsFeature, AuthorLargeEmbeddingsKeyFeature) override val adapter: LargeEmbeddingsAdapter = AuthorLargeEmbeddingsAdapter override val cacheType: hmf.Cache = hmf.Cache.AuthorLargeEmbeddings override val scopedStatsReceiver: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableLargeEmbeddingsFeatureHydrationParam) // Hashing Features override val defaultHashingFeatureParams: HashingFeatureParams = HashingFeatureParams( scales = Seq(3384241453L, 3372414709L), biases = Seq(1649585795L, 3131243219L), modulus = 3957384397L, bucketSize = 3000000L, ) override val modelName2HashingFeatureParams: Map[String, HashingFeatureParams] = Map( "hr_video_prod__v3_realtime" -> HashingFeatureParams( scales = Seq(113294449L, 601841083L), biases = Seq(2231299001L, 841367196L), modulus = 2343760591L, bucketSize = 3000000L, ), "hr_video_prod__v2_lembeds" -> HashingFeatureParams( scales = Seq(787140070L, 633713480L), biases = Seq(427768658L, 911091889L), modulus = 2888480981L, bucketSize = 300000L, ), "hr_prod__v4_embeds_230M" -> HashingFeatureParams( scales = Seq(371965780L, 328930218L), biases = Seq(139686260L, 37755056L), modulus = 631860353L, bucketSize = 30000000L, ), "hr_prod__v5_embeds_230M_and_transformer" -> HashingFeatureParams( scales = Seq(371965780L, 328930218L), biases = Seq(139686260L, 37755056L), modulus = 631860353L, bucketSize = 30000000L, ), "hr_prod__v5_watchtime" -> HashingFeatureParams( scales = Seq(2328530078L, 2844016377L), biases = Seq(1352496802L, 3011003330L), modulus = 3979826519L, bucketSize = 30000000L, ), "hr_prod__v6_transformer_v2" -> HashingFeatureParams( scales = Seq(371965780L, 328930218L), biases = Seq(139686260L, 37755056L), modulus = 631860353L, bucketSize = 30000000L, ), "hr_prod__v6_mixed_training" -> HashingFeatureParams( scales = Seq(371965780L, 328930218L), biases = Seq(139686260L, 37755056L), modulus = 631860353L, bucketSize = 30000000L, ), "hr_prod__v6_transformer_v2_kafka_merge_join" -> HashingFeatureParams( scales = Seq(371965780L, 328930218L), biases = Seq(139686260L, 37755056L), modulus = 631860353L, bucketSize = 30000000L, ), "hr_prod__v6_transformer_v2_realtime_debias_21apr" -> HashingFeatureParams( scales = Seq(371965780L, 328930218L), biases = Seq(139686260L, 37755056L), modulus = 631860353L, bucketSize = 30000000L, ), ) private val batchSize = 25 private def getBatchedFeatureMap( modelName: String, candidatesBatch: Seq[CandidateWithFeatures[TweetCandidate]], ): Future[Seq[FeatureMap]] = { val authorIds = candidatesBatch.map { candidate => candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L) } getLargeEmbeddings(authorIds, modelName).map { responses => responses.map { largeEmbeddingResponse => FeatureMapBuilder() .add(AuthorLargeEmbeddingsFeature, largeEmbeddingResponse.dataRecord) .add(AuthorLargeEmbeddingsKeyFeature, largeEmbeddingResponse.hashedKeys) .build() } } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val modelName = query.params(ModelNameParam) OffloadFuturePools.offloadBatchSeqToFutureSeq( candidates, getBatchedFeatureMap(modelName, _), batchSize ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/knuddels/jtokkit", "3rdparty/jvm/io/grpc:grpc-protobuf", "3rdparty/jvm/org/scalanlp:breeze", "hmli/hss/src/main/thrift/com/twitter/hss:thrift-scala", "home-mixer-features/thrift/src/main/thrift:thrift-java", "home-mixer-features/thrift/src/main/thrift:thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/gizmoduck_features", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/simclusters_features", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content", "joinkey/src/main/scala/com/twitter/joinkey/context", "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala", "language/thrift:types-lib-scala", "limiter/thrift-only/src/main/thrift:thrift-scala", "media-understanding/embeddings/src/main/thrift/com/twitter/media-understanding/embeddings:thrift-scala", "media-understanding/video-summary/thrift/src/main/thrift:thrift-scala", "periscope/api-proxy-thrift/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/location", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "representation-scorer/server/src/main/scala/com/twitter/representationscorer/common", "search/search-router/thrift/src/main/thrift:thrift-scala", "servo/repo/src/main/scala", "src/java/com/twitter/ml/api:api-base", "src/java/com/twitter/ml/api/constant", "src/java/com/twitter/search/common/util/lang", "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/simclusters_v2/common", "src/scala/com/twitter/suggests/controller_data", "src/scala/com/twitter/timelines/prediction/adapters/large_embeddings", "src/scala/com/twitter/timelines/prediction/adapters/real_graph", "src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph", "src/scala/com/twitter/timelines/prediction/adapters/request_context", "src/scala/com/twitter/timelines/prediction/adapters/twistly", "src/scala/com/twitter/timelines/prediction/adapters/two_hop_features", "src/scala/com/twitter/timelines/prediction/common/util", "src/scala/com/twitter/timelines/prediction/features/common", "src/scala/com/twitter/timelines/prediction/features/location", "src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph", "src/scala/com/twitter/timelines/prediction/features/simcluster", "src/scala/com/twitter/timelines/prediction/features/time_features", "src/scala/com/twitter/topic_recos/common", "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", "src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/timelines/content_understanding/user_interests:user_interests-scala", "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", "src/thrift/com/twitter/user_session_store:thrift-java", "strato/config/columns/analytics/video:video-strato-client", "strato/config/columns/audiencerewards/audienceRewardsService:getSuperFollowEligibility-strato-client", "strato/config/columns/content_understanding:content_understanding-strato-client", "strato/config/columns/content_understanding/internal/manhattan:manhattan-strato-client", "strato/config/columns/events/experiences/basketball:basketball-strato-client", "strato/config/columns/events/urt:urt-strato-client", "strato/config/columns/geo/service:service-strato-client", "strato/config/columns/heartbeat_optimizer:heartbeat_optimizer-strato-client", "strato/config/columns/hss/user_scores/api:api-strato-client", "strato/config/columns/language/tweet:tweet-strato-client", "strato/config/columns/language/user:user-strato-client", "strato/config/columns/ml/featureStore:featureStore-strato-client", "strato/config/columns/periscope:periscope-strato-client", "strato/config/columns/recommendations/interaction_graph/on_prem:on-prem-interaction-graph-user-features-strato-client", "strato/config/columns/recommendations/simclusters_v2:simclusters_v2-strato-client", "strato/config/columns/recommendations/user-signal-service:user-signal-service-strato-client", "strato/config/columns/searchai/grok:grok-strato-client", "strato/config/columns/searchai/storage:storage-strato-client", "strato/config/columns/subscription-services/subscription-verification:subscription-verification-strato-client", "strato/config/columns/trends/trip:trip-strato-client", "strato/config/columns/tweetypie/federated:federated-strato-client", "strato/config/columns/tweetypie/managed:managed-strato-client", "strato/config/columns/unified-counter/service:service-strato-client", "strato/config/columns/user-history-transformer/user-actions:user-actions-strato-client", "strato/config/columns/videoRecommendations/twitterClip:twitterClip-strato-client", "strato/config/columns/viewcounts:viewcounts-strato-client", "strato/config/src/thrift/com/twitter/strato/columns/heartbeat_optimizer:heartbeat_optimizer-scala", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence", "timelines/src/main/scala/com/twitter/timelines/clients/strato/topics", "timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", "topic-social-proof/server/src/main/scala/com/twitter/tsp/stores", "topic-social-proof/server/src/main/thrift:thrift-scala", "topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting", "tweetconvosvc/thrift/src/main/thrift:thrift-scala", "tweetsource/common/src/main/thrift:thrift-scala", "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", "user-signal-service/thrift/src/main/thrift:thrift-scala", "user_history_transformer/common/src/main/scala/com/twitter/user_history_transformer/common", "user_history_transformer/common/src/main/scala/com/twitter/user_history_transformer/util", "user_history_transformer/domain/src/main/scala:aggregation", "user_history_transformer/domain/src/main/scala:backbone", "user_history_transformer/domain/src/main/scala:user-history", "user_history_transformer/service/src/main/java/com/x/user_action_sequence", "user_history_transformer/thrift/src/main/thrift/com/x/user_action_sequence:user_action_sequence-scala", ], exports = [ "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BasketballContextFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.BasketballContextFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.param.HomeGlobalParams.BasketballTeamAccountIdsParam import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.timelines.render.{thriftscala => urt} import com.twitter.strato.catalog.Fetch import com.twitter.strato.generated.client.events.experiences.basketball.PostBasketballContextClientColumn import javax.inject.Inject import javax.inject.Singleton @Singleton class BasketballContextFeatureHydrator @Inject() ( postBasketballContextClientColumn: PostBasketballContextClientColumn ) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("BasketballContext") override val features: Set[Feature[_, _]] = Set(BasketballContextFeature) private val fetcher: Fetcher[Long, Unit, urt.BasketballContext] = postBasketballContextClientColumn.fetcher override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { val basketballAuthorIds = query.params(BasketballTeamAccountIdsParam) Stitch.collect { candidates.map { candidate => val servedType = candidate.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined) val isPromoted = (servedType == hmt.ServedType.ForYouPromoted || servedType == hmt.ServedType.FollowingPromoted) val authorId = candidate.features.getOrElse(AuthorIdFeature, None) val isBasketballAuthor = authorId.exists(id => basketballAuthorIds.contains(id)) // Skip hydration if the post is an ad or not from a basketball account if (isPromoted || !isBasketballAuthor) { Stitch.value(FeatureMapBuilder().add(BasketballContextFeature, None).build()) } else { fetcher.fetch(candidate.candidate.id, Unit).map { case Fetch.Result(Some(basketballContext), _) => FeatureMapBuilder().add(BasketballContextFeature, Some(basketballContext)).build() case _ => FeatureMapBuilder().add(BasketballContextFeature, None).build() } } } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BroadcastStateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.strato.catalog.Fetch import com.twitter.strato.client.Fetcher import com.twitter.strato.generated.client.periscope.CoreOnBroadcastClientColumn import com.twitter.ubs.{thriftscala => ubs} import javax.inject.Inject import javax.inject.Singleton object BroadcastStateFeature extends Feature[TweetCandidate, Option[ubs.BroadcastState]] @Singleton class BroadcastStateFeatureHydrator @Inject() ( coreOnBroadcastClientColumn: CoreOnBroadcastClientColumn) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with WithDefaultFeatureMap { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("BroadcastState") override val features: Set[Feature[_, _]] = Set(BroadcastStateFeature) private val pattern = "".r private val fetcher: Fetcher[String, Unit, ubs.Broadcast] = coreOnBroadcastClientColumn.fetcher override val defaultFeatureMap: FeatureMap = FeatureMap(BroadcastStateFeature, None) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { val broadcastIdMap = candidates.flatMap { candidate => candidate.features .getOrElse(TweetUrlsFeature, Seq.empty) .collectFirst { case pattern(broadcastId) => broadcastId } .map(broadcastId => candidate.candidate.id -> broadcastId) }.toMap val responses = Stitch.collect { broadcastIdMap.values.toSeq.distinct.map { broadcastId => fetcher.fetch(broadcastId).map { case Fetch.Result(Some(broadcast), _) if broadcast.broadcastId.nonEmpty => Some(broadcastId -> broadcast) case _ => None } } } responses.map { results => val broadcastMap = results.flatten.toMap candidates.map { candidate => val broadcastState = broadcastIdMap.get(candidate.candidate.id).flatMap { broadcastId => broadcastMap.get(broadcastId).flatMap(_.state) } FeatureMapBuilder().add(BroadcastStateFeature, broadcastState).build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/CategoryDiversityRescoringFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.simclusters_features.SimclustersFeaturesAdapter.SimclustersSparseTweetEmbeddingsFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.CategoryDiversityRescoringWeightParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.CategoryDiversityKParam import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import scala.jdk.CollectionConverters._ object CategoryDiversityRescoringFeatureHydrator extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("CategoryDiversityRescoring") override val features: Set[Feature[_, _]] = Set(ScoreFeature) final val EmptyDataRecord = new DataRecord() override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { diversityPenalty(query, candidates).map(score => FeatureMap(ScoreFeature, Some(score))) } private def diversityPenalty( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]], ): Seq[Double] = { val n = candidates.length val selected = scala.collection.mutable.Set[Int]() val newScores = Array.fill(n)(0.0) val weight = query.params(CategoryDiversityRescoringWeightParam) val k = query.params(CategoryDiversityKParam) val topKCategories = getCandidateCategory(k, candidates) val clusterCntMap = scala.collection.mutable.Map[String, Int]() val candidatesWithIndexWithCategories: Seq[ (CandidateWithFeatures[TweetCandidate], Int, Seq[String]) ] = candidates.zipWithIndex.zip(topKCategories).map { case ((candidate, index), categories) => (candidate, index, categories) } for (i <- 0 until n) { var maxScore = Double.NegativeInfinity var bestCandidateIndex = -1 var candidateCategories: Seq[String] = Seq.empty for ((candidate, index, categories) <- candidatesWithIndexWithCategories) { if (!selected.contains(index)) { val relevance = candidate.features.getOrElse(ScoreFeature, None).getOrElse(0.0) var penalty = 0.0 var totalCategoryCnt = 0 categories.foreach { category => val cnt = clusterCntMap.getOrElse(category, 0) penalty += math.log(cnt + 1) / math.log(2) } val score = math.max(relevance - weight * penalty, 0.00001) if (score > maxScore) { maxScore = score bestCandidateIndex = index candidateCategories = categories } } } if (bestCandidateIndex != -1) { selected += bestCandidateIndex newScores(bestCandidateIndex) = maxScore candidateCategories.foreach { category => clusterCntMap.put(category, clusterCntMap.getOrElse(category, 0) + 1) } } } newScores.toSeq } private def getCandidateCategory( k: Int, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Seq[String]] = { val topKClusters: Seq[Seq[String]] = candidates.map { candidate => val topKClustersMap = Option( candidate.features .getOrElse(SimClustersLogFavBasedTweetFeature, new DataRecord()) .getSparseContinuousFeatures ).flatMap(mapOpt => Option(mapOpt.get(SimclustersSparseTweetEmbeddingsFeature.getFeatureId))) topKClustersMap .map(_.asScala.toSeq.sortBy(-_._2).take(k).map(_._1)) .getOrElse(Seq.empty) } topKClusters } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ClipEmbeddingFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content.ClipEmbeddingFeaturesAdapter import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableClipEmbeddingFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableVideoClipEmbeddingFeatureHydrationDeciderParam import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.strato.generated.client.videoRecommendations.twitterClip.TwitterClipEmbeddingMhClientColumn import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ @Singleton class ClipEmbeddingFeatureHydrator @Inject() ( twitterClipEmbeddingMhClientColumn: TwitterClipEmbeddingMhClientColumn, statsReceiver: StatsReceiver) extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] with Logging { override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableClipEmbeddingFeaturesParam) && query.params(EnableVideoClipEmbeddingFeatureHydrationDeciderParam) override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ClipEmbedding") private val fetcher: Fetcher[Long, Unit, Seq[Double]] = twitterClipEmbeddingMhClientColumn.fetcher override val features: Set[Feature[_, _]] = Set(ClipEmbeddingFeature) private val DefaultFeatureMap = FeatureMapBuilder().add(ClipEmbeddingFeature, new DataRecord()).build() private val scopedStatsReceiver: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val clipEmbeddingNotFoundCounter = scopedStatsReceiver.counter("clipEmbeddingNotFound") private val clipEmbeddingFoundCounter = scopedStatsReceiver.counter("clipEmbeddingFound") override def apply( query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap ): Stitch[FeatureMap] = { fetcher .fetch(candidate.id).map { manhattanResult => manhattanResult.v match { case Some(embeddings) => clipEmbeddingFoundCounter.incr() val dataRecord = ClipEmbeddingFeaturesAdapter.adaptToDataRecords(embeddings).asScala.head FeatureMapBuilder().add(ClipEmbeddingFeature, dataRecord).build() case None => clipEmbeddingNotFoundCounter.incr() DefaultFeatureMap } }.onFailure(e => { error(s"Error fetching VideoClipEmbedding: $e") }) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ClipEmbeddingMediaUnderstandingFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content.ClipEmbeddingFeaturesAdapter import com.twitter.home_mixer.model.HomeFeatures.MediaCategoryFeature import com.twitter.home_mixer.model.HomeFeatures.MediaIdFeature import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableClipEmbeddingMediaUnderstandingFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableVideoClipEmbeddingMediaUnderstandingFeatureHydrationDeciderParam import com.twitter.media_understanding.embeddings.thriftscala.MediaEmbedding import com.twitter.media_understanding.embeddings.thriftscala.MediaEmbeddingInfo import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.client.Client import com.twitter.strato.client.Fetcher import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ object ClipEmbeddingFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class ClipEmbeddingMediaUnderstandingFeatureHydrator @Inject() ( stratoClient: Client, statsReceiver: StatsReceiver) extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] with Logging { override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableClipEmbeddingMediaUnderstandingFeaturesParam) && query.params(EnableVideoClipEmbeddingMediaUnderstandingFeatureHydrationDeciderParam) override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ClipEmbeddingMediaUnderstanding") val MediaModelName: String = "twitter_clip_256" val MediaVersion: String = "0" val mediaEmbeddingFetcher: Fetcher[(String, (String, String)), Unit, MediaEmbeddingInfo] = stratoClient.fetcher[(String, (String, String)), Unit, MediaEmbeddingInfo]( "media-understanding/embeddings/prod/embeddings" ) def getTweetMediaEmbeddings(key: String): Stitch[Option[Seq[Double]]] = { mediaEmbeddingFetchCounter.incr() mediaEmbeddingFetcher .fetch((key, (MediaModelName, MediaVersion))) .map { result => result.v match { case Some(mediaData) => mediaData.embedding match { case Some(MediaEmbedding.DoubleVector(doubles)) => mediaEmbeddingFetchSuccessCounter.incr() Some(doubles.map(_.toDouble)) case _ => mediaEmbeddingFetchFailureCounter.incr() None } case None => mediaEmbeddingFetchFailureCounter.incr() None } } .rescue { case e: Exception => mediaEmbeddingFetchFailureCounter.incr() Stitch.None } } def getClipEmbeddings( tweetId: Long, existingFeatures: FeatureMap ): Stitch[Option[Seq[Double]]] = { val mediaId = existingFeatures.getOrElse(MediaIdFeature, None) val mediaCategory = existingFeatures.getOrElse(MediaCategoryFeature, None) (mediaId, mediaCategory) match { case (Some(id), Some(category)) => val key = s"${category.getValue}/$id" getTweetMediaEmbeddings(key).map { case Some(embeddings) => Some(embeddings) case _ => clipEmbeddingNotFoundNoneCounter.incr() None } case _ => Stitch.value(None) } } override val features: Set[Feature[_, _]] = Set(ClipEmbeddingFeature) private val DefaultFeatureMap = FeatureMapBuilder().add(ClipEmbeddingFeature, new DataRecord()).build() private val scopedStatsReceiver: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val clipEmbeddingNotFoundNoneCounter = scopedStatsReceiver.counter("clipEmbeddingNotFoundNoneCounter") private val clipEmbeddingNotFoundCounter = scopedStatsReceiver.counter("clipEmbeddingNotFound") private val clipEmbeddingFoundCounter = scopedStatsReceiver.counter("clipEmbeddingFound") val mediaEmbeddingFetchCounter = statsReceiver.counter("media_embedding_fetch_total") val mediaEmbeddingFetchSuccessCounter = statsReceiver.counter("media_embedding_fetch_success") val mediaEmbeddingFetchFailureCounter = statsReceiver.counter("media_embedding_fetch_failure") override def apply( query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap ): Stitch[FeatureMap] = { getClipEmbeddings(candidate.id, existingFeatures) .map { case Some(embeddings) => clipEmbeddingFoundCounter.incr() val dataRecord = ClipEmbeddingFeaturesAdapter.adaptToDataRecords(embeddings).asScala.head FeatureMapBuilder().add(ClipEmbeddingFeature, dataRecord).build() case None => clipEmbeddingNotFoundCounter.incr() DefaultFeatureMap }.onFailure { case e: Exception => error(s"Error fetching VideoClipEmbedding: $e") } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ClipImageClusterIdFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.ClipImageClusterIdsFeature import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.ImageClipClusterIdInMemCache import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.cache.InProcessCache import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.videoRecommendations.twitterClip.TwitterClipImageClusterIdMh95ClientColumn import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class ClipImageClusterIdFeatureHydrator @Inject() ( twitterClipImageClusterIdMh95ClientColumn: TwitterClipImageClusterIdMh95ClientColumn, @Named(ImageClipClusterIdInMemCache) imageClipClusterIdInMemCache: InProcessCache[ Long, Option[Option[Long]] ], statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ClipImageClusterId") override val features: Set[Feature[_, _]] = Set(ClipImageClusterIdsFeature) private val clusterIdFetcher = twitterClipImageClusterIdMh95ClientColumn.fetcher private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") private val cacheHitCounter = scopedStatsReceiver.counter("cache/hit") private val cacheMissCounter = scopedStatsReceiver.counter("cache/miss") private val fetchExceptionCounter = scopedStatsReceiver.counter("getFromCacheOrFetch/exception") private def getFromCacheOrFetch(mediaId: Long): Stitch[Option[Option[Long]]] = { imageClipClusterIdInMemCache .get(mediaId) .map { cachedValue => cacheHitCounter.incr() Stitch.value(cachedValue) }.getOrElse { cacheMissCounter.incr() clusterIdFetcher .fetch(mediaId) .map(_.v) .liftToOption() .flatMap { clusterIdOpt => imageClipClusterIdInMemCache.set(mediaId, clusterIdOpt) Stitch.value(clusterIdOpt) } }.handle { case _: Exception => fetchExceptionCounter.incr() None } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { Stitch.collect( candidates.map { candidate => val mediaIds = candidate.features.getOrElse(TweetMediaIdsFeature, Seq.empty[Long]) val items: Seq[Stitch[Option[(Long, Long)]]] = mediaIds.map { mediaId => getFromCacheOrFetch(mediaId).map { case Some(Some(mediaClusterId)) => keyFoundCounter.incr() Some(mediaId -> mediaClusterId) case _ => keyNotFoundCounter.incr() None } } Stitch.collect(items).map { results => val mediaClusterMap = results.flatten.toMap FeatureMap(ClipImageClusterIdsFeature, mediaClusterMap) } } ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DependentBulkCandidateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch case class DependentBulkCandidateFeatureHydrator[Query <: PipelineQuery]( parentCandidateFeatureHydrator: BulkCandidateFeatureHydrator[Query, TweetCandidate], childrenCandidateFeatureHydrators: Seq[ BulkCandidateFeatureHydrator[Query, TweetCandidate] with WithDefaultFeatureMap ]) extends BulkCandidateFeatureHydrator[Query, TweetCandidate] with Conditionally[Query] { override final val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "Dependent" + parentCandidateFeatureHydrator.identifier.name) override def onlyIf(query: Query): Boolean = { parentCandidateFeatureHydrator match { case candidateHydrator: BulkCandidateFeatureHydrator[_, _] with Conditionally[Query] => candidateHydrator.onlyIf(query) case _ => true } } override val features: Set[Feature[_, _]] = childrenCandidateFeatureHydrators.foldLeft(parentCandidateFeatureHydrator.features) { (features, childFeatureHydrator) => features ++ childFeatureHydrator.features } override def apply( query: Query, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { val parentFeatureMapsStitch = parentCandidateFeatureHydrator.apply(query, candidates) parentFeatureMapsStitch.flatMap { parentFeatureMaps => val updatedCandidates = candidates.zip(parentFeatureMaps).map { case (tweetCandidate, featureMap) => new CandidateWithFeatures[TweetCandidate] { override val candidate = tweetCandidate.candidate override val features = tweetCandidate.features ++ featureMap } } val (childrenCandidateFeatureHydratorsFiltered, childrenCandidateFeatureHydratorsDefault) = childrenCandidateFeatureHydrators.partition { case candidateFeatureHydrator: BulkCandidateFeatureHydrator[_, _] with Conditionally[ Query ] => candidateFeatureHydrator.onlyIf(query) case _: BulkCandidateFeatureHydrator[_, _] => true case _ => false } val childrenFeatureMapsDefault = childrenCandidateFeatureHydratorsDefault.map { featureHydrator => Seq.fill(candidates.size)(featureHydrator.defaultFeatureMap) } val childrenFeatureMapsStitch = Stitch.traverse(childrenCandidateFeatureHydratorsFiltered) { _.apply(query, updatedCandidates) } childrenFeatureMapsStitch.map { childrenFeatureMaps => val allFeatureMaps = (parentFeatureMaps +: childrenFeatureMaps) ++ childrenFeatureMapsDefault allFeatureMaps.transpose.map(FeatureMap.merge(_)) } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DismissInfoQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.timelinemixer.clients.manhattan.InjectionHistoryClient import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelinemixer.clients.manhattan.DismissInfo import com.twitter.timelineservice.suggests.thriftscala.SuggestType import javax.inject.Inject import javax.inject.Singleton object DismissInfoQueryFeatureHydrator { val DismissInfoSuggestTypes = Seq(SuggestType.WhoToFollow) } @Singleton case class DismissInfoQueryFeatureHydrator @Inject() ( dismissInfoClient: InjectionHistoryClient) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("DismissInfo") override val features: Set[Feature[_, _]] = Set(DismissInfoFeature) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = Stitch.callFuture { dismissInfoClient .readDismissInfoEntries( query.getRequiredUserId, DismissInfoQueryFeatureHydrator.DismissInfoSuggestTypes).map { response => val dismissInfoMap = response.mapValues(DismissInfo.fromThrift) FeatureMapBuilder().add(DismissInfoFeature, dismissInfoMap).build() } } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8, 50, 60, 60) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/DiversityRescoringFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import breeze.linalg._ import breeze.numerics.sqrt //import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinEmbeddingsFeatures.TwhinTweetEmbeddingsFeature import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.TransformerEmbeddingsFeatures.PostTransformerEmbeddingsJointBlueFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.TwhinDiversityRescoringWeightParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.TwhinDiversityRescoringRatioParam import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import scala.jdk.CollectionConverters.collectionAsScalaIterableConverter object DiversityRescoringFeatureHydrator extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("DiversityRescoring") override val features: Set[Feature[_, _]] = Set(ScoreFeature) final val EmptyDataRecord = new DataRecord() private val embeddingsSize = 128 private val defaultEmbeddings = Seq.fill(embeddingsSize)(0.0) private val defaultDenseVector = DenseVector(Array.fill(embeddingsSize)(1.0)) / sqrt(embeddingsSize) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { val embeddings = candidates.map { candidate => val embeddingsTensor = Option( candidate.features .getOrElse(TransformerPostEmbeddingJointBlueFeature, EmptyDataRecord) .getTensors) .flatMap(tensorsOpt => Option(tensorsOpt.get(PostTransformerEmbeddingsJointBlueFeature.getFeatureId))) embeddingsTensor .map(_.getFloatTensor.floats.asScala.map(_.doubleValue).toSeq) .getOrElse(defaultEmbeddings) } val denseEmbeddingsNormalized = embeddings.map { seq => val denseVector = DenseVector(seq.toArray) val normVal = norm(denseVector) if (normVal != 0) denseVector / normVal else defaultDenseVector } val distanceMatrix = DenseMatrix.zeros[Double](denseEmbeddingsNormalized.length, denseEmbeddingsNormalized.length) for (i <- denseEmbeddingsNormalized.indices; j <- denseEmbeddingsNormalized.indices) { distanceMatrix(i, j) = norm(denseEmbeddingsNormalized(i) - denseEmbeddingsNormalized(j)) } mmr(query, candidates, distanceMatrix).map(score => FeatureMap(ScoreFeature, Some(score))) } def mmr( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]], distanceMatrix: DenseMatrix[Double] ): Seq[Double] = { val n = candidates.length val diversityRatio = query.params(TwhinDiversityRescoringRatioParam) val diversityWeight = query.params(TwhinDiversityRescoringWeightParam) val selected = scala.collection.mutable.Set[Int]() val newScores = Array.fill(n)(0.0) val candidatesWithIndex = candidates.zipWithIndex for (i <- 0 until n) { var maxScore = Double.NegativeInfinity var bestCandidateIndex = -1 for ((candidate, index) <- candidatesWithIndex) { if (!selected.contains(index)) { val relevance = candidate.features.getOrElse(ScoreFeature, None).getOrElse(0.0) val minDistance = { if (selected.isEmpty || selected.size < (1 - diversityRatio) * n) None else { selected .map(j => distanceMatrix(index, j)) .reduceOption { (a, b) => if (a < b) a else b } } } val score = relevance + diversityWeight * minDistance.getOrElse(2.0) if (score > maxScore) { maxScore = score bestCandidateIndex = index } } } if (bestCandidateIndex != -1) { selected += bestCandidateIndex newScores(bestCandidateIndex) = maxScore } } newScores.toSeq } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/EarlybirdSearchResultFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdSearchResultFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRepository import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import com.twitter.util.Return import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class EarlybirdSearchResultFeatureHydrator @Inject() ( @Named(EarlybirdRepository) client: KeyValueRepository[ (Seq[Long], Long), Long, eb.ThriftSearchResult ], override val statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("EarlybirdSearchResult") override val features: Set[Feature[_, _]] = Set( EarlybirdScoreFeature, EarlybirdSearchResultFeature ) override val statScope: String = identifier.toString override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { client((candidates.map(_.candidate.id), query.getRequiredUserId)) .map(handleResponse(candidates, _)) } private def handleResponse( candidates: Seq[CandidateWithFeatures[TweetCandidate]], results: KeyValueResult[Long, eb.ThriftSearchResult] ): Seq[FeatureMap] = { candidates .map { candidate => observedGet(Some(candidate.candidate.id), results) }.map { case Return(Some(searchResult)) => FeatureMapBuilder() .add(EarlybirdScoreFeature, searchResult.metadata.flatMap(_.score)) .add(EarlybirdSearchResultFeature, Some(searchResult)) .build() case other => FeatureMapBuilder() .add(EarlybirdScoreFeature, None) .add(EarlybirdSearchResultFeature, other) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature import com.twitter.home_mixer.model.HomeFeatures.HasRecentFeedbackSinceCacheTtlFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelinemixer.clients.feedback.FeedbackHistoryManhattanClient import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton @Singleton case class FeedbackHistoryQueryFeatureHydrator @Inject() ( feedbackHistoryClient: FeedbackHistoryManhattanClient) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory") override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature, HasRecentFeedbackSinceCacheTtlFeature) override def hydrate( query: PipelineQuery ): Stitch[FeatureMap] = Stitch .callFuture(feedbackHistoryClient.get(query.getRequiredUserId)) .map { feedbackHistory => val latestFeedbackTimestamp = if (feedbackHistory.nonEmpty) Some(feedbackHistory.map(_.timestamp.inMilliseconds).max) else None val cachedScoredTweetsTtl = query.params(CachedScoredTweets.TTLParam) val hasRecentFeedbackSinceCacheTtl = latestFeedbackTimestamp match { case Some(timestamp) => Time.fromMilliseconds(timestamp).untilNow < cachedScoredTweetsTtl case None => false } val featureMapBuilder = FeatureMapBuilder() .add(FeedbackHistoryFeature, feedbackHistory) .add(HasRecentFeedbackSinceCacheTtlFeature, hasRecentFeedbackSinceCacheTtl) featureMapBuilder.build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FollowableUttTopicsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.topic_recos.common.LocaleUtil import com.twitter.topiclisting.SemanticCoreEntityId import com.twitter.topiclisting.TopicListingViewerContext import com.twitter.tsp.stores.UttTopicFilterStore import com.twitter.tsp.{thriftscala => tsp} import javax.inject.Inject import javax.inject.Singleton object FollowableUttTopicsFeatures extends Feature[PipelineQuery, Option[Map[SemanticCoreEntityId, Option[tsp.TopicFollowType]]]] @Singleton class FollowableUttTopicsQueryFeatureHydrator @Inject() ( uttStore: UttTopicFilterStore, override val statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FollowableUttTopics") override val features: Set[Feature[_, _]] = Set(FollowableUttTopicsFeatures) override val statScope: String = identifier.toString override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val context = TopicListingViewerContext.fromClientContext(query.clientContext) Stitch.callFuture { uttStore .getAllowListTopicsForUser( userId = query.getRequiredUserId, topicListingSetting = tsp.TopicListingSetting.Followable, context = context .copy(languageCode = LocaleUtil.getStandardLanguageCode(context.languageCode)), bypassModes = None ).map { topics => FeatureMapBuilder().add(FollowableUttTopicsFeatures, Some(topics)).build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.follow_recommendations.{thriftscala => frs} import com.twitter.product_mixer.component_library.candidate_source.recommendations.UserFollowRecommendationsCandidateSource import com.twitter.product_mixer.component_library.candidate_source.recommendations.CachedUserFollowRecommendationsCandidateSource import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton object FrsSeedUserIdsFeature extends Feature[TweetCandidate, Option[Seq[Long]]] object FrsUserToFollowedByUserIdsFeature extends Feature[TweetCandidate, Map[Long, Seq[Long]]] @Singleton case class FrsSeedUsersQueryFeatureHydrator @Inject() ( userFollowRecommendationsCandidateSource: UserFollowRecommendationsCandidateSource, cachedUserFollowRecommendationsCandidateSource: CachedUserFollowRecommendationsCandidateSource) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FrsSeedUsers") private val maxUsersToFetch = 100 override val features: Set[Feature[_, _]] = Set( FrsSeedUserIdsFeature, FrsUserToFollowedByUserIdsFeature ) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val frsRequest = frs.RecommendationRequest( clientContext = frs.ClientContext(query.getOptionalUserId), displayLocation = frs.DisplayLocation.HomeTimelineTweetRecs, maxResults = Some(maxUsersToFetch) ) userFollowRecommendationsCandidateSource(StratoKeyView(frsRequest, Unit)) .map { userRecommendations: Seq[frs.UserRecommendation] => val seedUserIds = userRecommendations.map(_.userId) val seedUserIdsSet = seedUserIds.toSet val userToFollowedByUserIds: Map[Long, Seq[Long]] = userRecommendations.flatMap { userRecommendation => if (seedUserIdsSet.contains(userRecommendation.userId)) { val followProof = userRecommendation.reason.flatMap(_.accountProof).flatMap(_.followProof) val followedByUserIds = followProof.map(_.userIds).getOrElse(Seq.empty) Some(userRecommendation.userId -> followedByUserIds) } else { None } }.toMap FeatureMapBuilder() .add(FrsSeedUserIdsFeature, Some(seedUserIds)) .add(FrsUserToFollowedByUserIdsFeature, userToFollowedByUserIds) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GeoduckAuthorLocationHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.dal.personal_data.{thriftjava => pd} import com.twitter.geoduck.common.thriftscala.TransactionLocation import com.twitter.geoduck.common.{thriftscala => t} import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.product_mixer.component_library.feature.location.Location import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature import com.twitter.product_mixer.core.feature.datarecord.LongDiscreteDataRecordCompatible import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.cache.ExpiringLruInProcessCache import com.twitter.servo.cache.InProcessCache import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.geo.service.UserLocationClientColumn import com.twitter.timelines.prediction.features.location.LocationFeatures import javax.inject.Inject import javax.inject.Singleton object AuthorLocationNeighborhoodFeature extends DataRecordOptionalFeature[PipelineQuery, Long] with LongDiscreteDataRecordCompatible { override val featureName: String = LocationFeatures.AUTHOR_LOCATION_NEIGHBORDHOOD.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.InferredLocation) } object AuthorLocationCityFeature extends DataRecordOptionalFeature[PipelineQuery, Long] with LongDiscreteDataRecordCompatible { override val featureName: String = LocationFeatures.AUTHOR_LOCATION_CITY.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.InferredLocation) } object AuthorLocationMetroFeature extends DataRecordOptionalFeature[PipelineQuery, Long] with LongDiscreteDataRecordCompatible { override val featureName: String = LocationFeatures.AUTHOR_LOCATION_METRO.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.InferredLocation) } object AuthorLocationRegionFeature extends DataRecordOptionalFeature[PipelineQuery, Long] with LongDiscreteDataRecordCompatible { override val featureName: String = LocationFeatures.AUTHOR_LOCATION_REGION.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.InferredLocation) } object AuthorLocationCountryFeature extends DataRecordOptionalFeature[PipelineQuery, Long] with LongDiscreteDataRecordCompatible { override val featureName: String = LocationFeatures.AUTHOR_LOCATION_COUNTRY.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.InferredLocation) } object GeoduckAuthorLocationHydrator { private val BaseTTLMinutes = 60 * 24 private val TTL = (BaseTTLMinutes + scala.util.Random.nextInt(60)).minutes val cache: InProcessCache[Long, Option[TransactionLocation]] = new ExpiringLruInProcessCache[Long, Option[TransactionLocation]]( ttl = TTL, maximumSize = 150 * 1000 // Cache up to 150k users ) } @Singleton class GeoduckAuthorLocationHydrator @Inject() ( userLocationClientColumn: UserLocationClientColumn) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "GeoduckAuthorLocationHydrator") override val features: Set[Feature[_, _]] = Set( LocationFeature, AuthorLocationNeighborhoodFeature, AuthorLocationCityFeature, AuthorLocationMetroFeature, AuthorLocationRegionFeature, AuthorLocationCountryFeature ) private val PlaceQuery = t.PlaceQuery( placeTypes = Some( Set( t.PlaceType.Neighborhood, t.PlaceType.City, t.PlaceType.Metro, t.PlaceType.Admin1, t.PlaceType.Country ) ) ) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)).distinct val (hitIds, missIds) = authorIds.partition(id => GeoduckAuthorLocationHydrator.cache .get(id) .isDefined) val cachedOptionMap: Map[Long, Option[TransactionLocation]] = hitIds.map(id => id -> GeoduckAuthorLocationHydrator.cache.get(id).get).toMap val fetchCacheMisses: Stitch[Map[Long, TransactionLocation]] = if (missIds.isEmpty) Stitch.value(Map.empty[Long, TransactionLocation]) else { userLocationClientColumn.fetcher .fetch( key = Unit, t.UserLocationRequest( userIds = missIds, placeQuery = Some(PlaceQuery) ) ) .map { response => response.v.toList.flatMap(_._1).toMap } .handle { case _ => Map.empty[Long, TransactionLocation] } } val allLocationsStitch: Stitch[Map[Long, TransactionLocation]] = fetchCacheMisses.map { fetchedMap => missIds.foreach { id => val locOpt: Option[TransactionLocation] = fetchedMap.get(id) GeoduckAuthorLocationHydrator.cache.set(id, locOpt) } val cachedLocs: Map[Long, TransactionLocation] = cachedOptionMap.collect { case (id, Some(loc)) => id -> loc } cachedLocs ++ fetchedMap } allLocationsStitch.map { allLocations => candidates.map { candidate => val locOpt = for { authorId <- candidate.features.getOrElse(AuthorIdFeature, None) loc <- allLocations.get(authorId) } yield loc locOpt .map { transactionLocation => val placeMap = transactionLocation.placeMap val locationDetails = Location( neighborhood = placeMap .flatMap(_.get(t.PlaceType.Neighborhood)) .flatMap(_.headOption), city = placeMap.flatMap(_.get(t.PlaceType.City)).flatMap(_.headOption), metro = placeMap.flatMap(_.get(t.PlaceType.Metro)).flatMap(_.headOption), region = placeMap.flatMap(_.get(t.PlaceType.Admin1)).flatMap(_.headOption), country = placeMap.flatMap(_.get(t.PlaceType.Country)).flatMap(_.headOption) ) FeatureMapBuilder() .add(LocationFeature, Option(locationDetails)) .add(AuthorLocationNeighborhoodFeature, locationDetails.neighborhood) .add(AuthorLocationCityFeature, locationDetails.city) .add(AuthorLocationMetroFeature, locationDetails.metro) .add(AuthorLocationRegionFeature, locationDetails.region) .add(AuthorLocationCountryFeature, locationDetails.country) .build() } .getOrElse { FeatureMapBuilder() .add(LocationFeature, None) .add(AuthorLocationNeighborhoodFeature, None) .add(AuthorLocationCityFeature, None) .add(AuthorLocationMetroFeature, None) .add(AuthorLocationRegionFeature, None) .add(AuthorLocationCountryFeature, None) .build() } } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckAuthorFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.github.nscala_time.time.Imports.LocalDate import com.twitter.ads.entities.db.{thriftscala => ae} import com.twitter.conversions.DurationOps._ import com.twitter.gizmoduck.{thriftscala => gt} import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.gizmoduck_features.GFeatures import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.gizmoduck_features.GizmoduckFeaturesAdapter import com.twitter.home_mixer.model.HomeFeatures.AuthorAccountAge import com.twitter.home_mixer.model.HomeFeatures.AuthorFollowersFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIsProtectedFeature import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.IsSupportAccountReplyFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorSafetyLabels import com.twitter.home_mixer.module.SupportAccountsConfig import com.twitter.home_mixer.param.HomeMixerInjectionNames.GizmoduckTimelinesCache import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.cache.TtlCache import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.util.Duration import com.twitter.util.Future import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ private case class UserFeatures( isBlueVerified: Boolean, isVerifiedOrganization: Boolean, isVerifiedOrganizationAffiliate: Boolean, isProtected: Boolean, isSupportAccount: Boolean, followersCount: Option[Long], accountAge: Option[Duration], labels: Option[Seq[String]]) extends GFeatures object GizmoduckAuthorFeatures extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class GizmoduckAuthorFeatureHydrator @Inject() ( gizmoduck: gt.UserService.MethodPerEndpoint, @Named(GizmoduckTimelinesCache) cacheClient: TtlCache[Long, gt.User], supportAccounts: SupportAccountsConfig) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GizmoduckAuthor") override val features: Set[Feature[_, _]] = Set( AuthorAccountAge, AuthorFollowersFeature, AuthorIsBlueVerifiedFeature, AuthorIsProtectedFeature, IsSupportAccountReplyFeature, GizmoduckAuthorFeatures, AuthorSafetyLabels ) private val CacheTTL = 24.hours private val queryFields: Set[gt.QueryFields] = Set( gt.QueryFields.AdvertiserAccount, gt.QueryFields.Profile, gt.QueryFields.Safety, gt.QueryFields.Labels, gt.QueryFields.Counts ) private val lookupContext = gt.LookupContext(isRequestSheddable = Some(true)) // Advertiser service levels which are assumed to provide support via replies private val AdvertiserServiceLevels = Set[ae.ServiceLevel]( ae.ServiceLevel.Dso, ae.ServiceLevel.Mms, ae.ServiceLevel.Reseller, ae.ServiceLevel.Smb ) private val SupportAccounts = supportAccounts.accounts private val DefaultUserFeatures = UserFeatures( isBlueVerified = false, isVerifiedOrganization = false, isVerifiedOrganizationAffiliate = false, isProtected = false, isSupportAccount = false, followersCount = None, accountAge = None, labels = None ) private def extractFeaturesFromUser( user: gt.User ): UserFeatures = { val isBlueVerified = user.safety.flatMap(_.isBlueVerified).getOrElse(false) val safetyLabels = user.labels.map(_.labels.map(_.labelValue.name)) val verifiedOrganizationDetails = user.safety.flatMap(_.verifiedOrganizationDetails) val isVerifiedOrganization = verifiedOrganizationDetails.flatMap(_.isVerifiedOrganization).getOrElse(false) val isVerifiedOrganizationAffiliate = verifiedOrganizationDetails.flatMap(_.isVerifiedOrganizationAffiliate).getOrElse(false) val isProtected = user.safety.exists(_.isProtected) val isSupportAccount = user.profile.exists(_.businessProfileState == gt.BusinessProfileState.Enabled) || (user.advertiserAccount.flatMap(_.advertiserType).nonEmpty && user.advertiserAccount .flatMap(_.serviceLevels).getOrElse(Seq.empty) .exists(AdvertiserServiceLevels.contains)) val followersCount = user.counts.map(_.followers) val accountAge = Duration.fromSeconds( (LocalDate.now().toDate.toInstant.getEpochSecond - (user.createdAtMsec / 1000)).toInt) UserFeatures( isBlueVerified, isVerifiedOrganization, isVerifiedOrganizationAffiliate, isProtected, isSupportAccount, followersCount, Some(accountAge), safetyLabels ) } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)).distinct cacheClient .get(authorIds) .flatMap { cacheResponse => val cacheHydratedUsersMap = cacheResponse.found val notFoundUsers = cacheResponse.notFound.toSeq.distinct val gizmoduckResponseFuture = if (notFoundUsers.nonEmpty) { gizmoduck.get(lookupContext, notFoundUsers, queryFields) } else Future.value(Seq.empty) gizmoduckResponseFuture.map { gizmoduckResponse => val gizmoduckHydratedUsersMap = gizmoduckResponse.collect { case userResult if userResult.user.isDefined => val user = userResult.user.get cacheClient.add(user.id, user, CacheTTL) user.id -> user }.toMap val hydratedUsersMap = (cacheHydratedUsersMap ++ gizmoduckHydratedUsersMap) .mapValues(extractFeaturesFromUser) candidates.map { candidate => val authorIdOpt = candidate.features.getOrElse(AuthorIdFeature, None) val userFeatures = authorIdOpt.flatMap(hydratedUsersMap.get).getOrElse(DefaultUserFeatures) // Some accounts run promotions and send replies automatically. // We assume that a reply that took more than one minute is not an auto-reply. // If time difference doesn't exist, this means that one of the tweets was // not snowflake and therefore much older, and therefore OK as an extended reply. val timeDifference = candidate.features.getOrElse(InReplyToTweetIdFeature, None).map { SnowflakeId.timeFromId(candidate.candidate.id) - SnowflakeId.timeFromId(_) } val isAutoReply = timeDifference.exists(_ < 1.minute) val isSupportAccountReply = candidate.features.getOrElse(InReplyToTweetIdFeature, None).nonEmpty && !candidate.features.getOrElse(IsRetweetFeature, false) && candidate.features.getOrElse(FromInNetworkSourceFeature, false) && (userFeatures.isSupportAccount || authorIdOpt.exists(SupportAccounts.contains) || isAutoReply) val gizmoduckFeaturesRichDataRecords = GizmoduckFeaturesAdapter.adaptToDataRecords(userFeatures).asScala.head FeatureMapBuilder() .add(AuthorAccountAge, userFeatures.accountAge) .add(AuthorFollowersFeature, userFeatures.followersCount) .add(AuthorIsBlueVerifiedFeature, userFeatures.isBlueVerified) .add(AuthorIsProtectedFeature, userFeatures.isProtected) .add(IsSupportAccountReplyFeature, isSupportAccountReply) .add(GizmoduckAuthorFeatures, gizmoduckFeaturesRichDataRecords) .add(AuthorSafetyLabels, userFeatures.labels) .build() } } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckUserQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.gizmoduck.{thriftscala => gt} import com.twitter.home_mixer.model.HomeFeatures.SignupCountryFeature import com.twitter.home_mixer.model.HomeFeatures.SignupSourceFeature import com.twitter.home_mixer.model.HomeFeatures.UserFollowersCountFeature import com.twitter.home_mixer.model.HomeFeatures.UserFollowingCountFeature import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature import com.twitter.home_mixer.model.HomeFeatures.UserTypeFeature import com.twitter.home_mixer.model.HomeFeatures.ViewerAllowsAdsPersonalizationFeature import com.twitter.home_mixer.model.HomeFeatures.ViewerAllowsDataSharingFeature import com.twitter.home_mixer.model.HomeFeatures.ViewerAllowsForYouRecommendationsFeature import com.twitter.home_mixer.model.HomeFeatures.ViewerHasPremiumTier import com.twitter.home_mixer.model.HomeFeatures.ViewerSafetyLabels import com.twitter.home_mixer.model.signup.MarchMadness import com.twitter.home_mixer.model.signup.Onboard import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.stitch.gizmoduck.Gizmoduck import javax.inject.Inject import javax.inject.Singleton @Singleton case class GizmoduckUserQueryFeatureHydrator @Inject() (gizmoduck: Gizmoduck) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GizmoduckUser") override val features: Set[Feature[_, _]] = Set( UserFollowingCountFeature, UserFollowersCountFeature, UserTypeFeature, UserScreenNameFeature, ViewerHasPremiumTier, SignupCountryFeature, SignupSourceFeature, ViewerAllowsForYouRecommendationsFeature, ViewerAllowsDataSharingFeature, ViewerAllowsAdsPersonalizationFeature, ViewerSafetyLabels ) private val queryFields: Set[gt.QueryFields] = Set( gt.QueryFields.Counts, gt.QueryFields.Safety, gt.QueryFields.Profile, gt.QueryFields.Account, gt.QueryFields.Labels ) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId gizmoduck .getUserById( userId = userId, queryFields = queryFields, context = gt.LookupContext(forUserId = Some(userId), includeSoftUsers = true)) .map { user => val premiumTier = user.safety .map { safety => safety.skipRateLimit.contains(true) || safety.isBlueVerified.contains(true) || safety.verifiedType.contains(gt.VerifiedType.Business) || safety.verifiedType.contains(gt.VerifiedType.Government) || safety.verifiedOrganizationDetails.exists(_.isVerifiedOrganization.getOrElse(false)) || safety.verifiedOrganizationDetails .exists(_.isVerifiedOrganizationAffiliate.contains(true)) }.getOrElse(false) val signupSource = user.safety.flatMap(_.signupCreationSource).flatMap { case gt.SignupCreationSource.MarchMadness => Some(MarchMadness) case gt.SignupCreationSource.Onboard => Some(Onboard) case _ => None } FeatureMapBuilder() .add(UserFollowingCountFeature, user.counts.map(_.following.toInt)) .add(UserFollowersCountFeature, user.counts.map(_.followers.toInt)) .add(UserTypeFeature, Some(user.userType)) .add(UserScreenNameFeature, user.profile.map(_.screenName)) .add(ViewerHasPremiumTier, premiumTier) .add(SignupCountryFeature, user.safety.flatMap(_.signupCountryCode)) .add(SignupSourceFeature, signupSource) .add( ViewerAllowsForYouRecommendationsFeature, user.account.flatMap(_.allowForYouRecommendations) ) .add( ViewerAllowsDataSharingFeature, user.account.map(_.allowSharingDataForThirdPartyPersonalization) ).add( ViewerAllowsAdsPersonalizationFeature, user.account.map(_.allowAdsPersonalization) ).add( ViewerSafetyLabels, user.labels.map(_.labels.map(_.labelValue.name)) ) .build() } } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.7) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GraphTwoHopFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.graph_feature_service.{thriftscala => gfs} import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.GraphTwoHopRepository import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.two_hop_features.TwoHopFeaturesAdapter import com.twitter.util.Try import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object GraphTwoHopFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class GraphTwoHopFeatureHydrator @Inject() ( @Named(GraphTwoHopRepository) client: KeyValueRepository[ (Seq[Long], Long), Long, Seq[gfs.IntersectionValue] ], override val statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GraphTwoHop") override val features: Set[Feature[_, _]] = Set(GraphTwoHopFeature, FollowedByUserIdsFeature) override val statScope: String = identifier.toString private val twoHopFeaturesAdapter = new TwoHopFeaturesAdapter private val FollowFeatureType = gfs.FeatureType(gfs.EdgeType.Following, gfs.EdgeType.FollowedBy) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { // Apply filters to in network candidates for retweets only. val (inNetworkCandidates, oonCandidates) = candidates.partition { candidate => candidate.features.getOrElse(FromInNetworkSourceFeature, false) } val inNetworkCandidatesToHydrate = inNetworkCandidates.filter(_.features.getOrElse(IsRetweetFeature, false)) val candidatesToHydrate = (inNetworkCandidatesToHydrate ++ oonCandidates) .flatMap(candidate => CandidatesUtil.getOriginalAuthorId(candidate.features)).distinct val response = client((candidatesToHydrate, query.getRequiredUserId)) response.map { result => candidates.map { candidate => val originalAuthorId = CandidatesUtil.getOriginalAuthorId(candidate.features) val value = observedGet(key = originalAuthorId, keyValueResult = result) val transformedValue = postTransformer(value) val followedByUserIds = value.toOption .flatMap(getFollowedByUserIds(_)) .getOrElse(Seq.empty) FeatureMapBuilder() .add(GraphTwoHopFeature, transformedValue) .add(FollowedByUserIdsFeature, followedByUserIds) .build() } } } private def getFollowedByUserIds(input: Option[Seq[gfs.IntersectionValue]]): Option[Seq[Long]] = input.map { _.filter(_.featureType == FollowFeatureType).flatMap(_.intersectionIds).flatten.distinct } private def postTransformer(input: Try[Option[Seq[gfs.IntersectionValue]]]): Try[DataRecord] = input.map(twoHopFeaturesAdapter.adaptToDataRecords(_).asScala.head) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GrokAnnotationsFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.GrokTopics.GrokCategoryIdToNameMap import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature import com.twitter.home_mixer.model.HomeFeatures.GrokAnnotationsFeature import com.twitter.home_mixer.model.HomeFeatures.GrokCategoryDataRecordFeature import com.twitter.home_mixer.model.HomeFeatures.GrokTopCategoryFeature import com.twitter.home_mixer.model.HomeFeatures.GrokIsGoreFeature import com.twitter.home_mixer.model.HomeFeatures.GrokIsLowQualityFeature import com.twitter.home_mixer.model.HomeFeatures.GrokIsNsfwFeature import com.twitter.home_mixer.model.HomeFeatures.GrokIsOcrFeature import com.twitter.home_mixer.model.HomeFeatures.GrokIsSpamFeature import com.twitter.home_mixer.model.HomeFeatures.GrokIsViolentFeature import com.twitter.home_mixer.model.HomeFeatures.GrokPoliticalInclinationFeature import com.twitter.home_mixer.model.HomeFeatures.GrokSlopScoreFeature import com.twitter.home_mixer.model.HomeFeatures.GrokSunnyScoreFeature import com.twitter.home_mixer.model.HomeFeatures.GrokTagsFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableGrokAnnotations import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer_features.{thriftscala => hmf} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton @Singleton class GrokAnnotationsFeatureHydrator @Inject() ( homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GrokAnnotations") override val features: Set[Feature[_, _]] = Set( GrokAnnotationsFeature, GrokCategoryDataRecordFeature, GrokTopCategoryFeature, GrokTagsFeature, GrokIsGoreFeature, GrokIsNsfwFeature, GrokIsSpamFeature, GrokIsViolentFeature, GrokIsLowQualityFeature, GrokIsOcrFeature, GrokSunnyScoreFeature, GrokPoliticalInclinationFeature, GrokSlopScoreFeature, DebugStringFeature ) override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableGrokAnnotations) private val batchSize = 64 private def getGrokAnnotationsFromHMF( tweetIdsToHydrate: Seq[Long], ): Future[Seq[Option[hmt.GrokAnnotations]]] = { val keySerialized = tweetIdsToHydrate.map(_.toString) val request = hmf.HomeMixerFeaturesRequest( keySerialized, hmf.Cache.GrokPostAnnotations ) val responseFut = homeMixerFeatureService.getHomeMixerFeatures(request) responseFut .map { response => response.homeMixerFeatures .map { homeMixerFeaturesOpt => homeMixerFeaturesOpt.homeMixerFeaturesType.map { case hmf.HomeMixerFeaturesType.GrokPostAnnotations(data) => val metadata = data.annotations.tweetBoolMetadata.map { metadata => hmt.GrokMetadata( isNsfw = metadata.isNsfw.getOrElse(false), isGore = metadata.isGore.getOrElse(false), isViolent = metadata.isViolent.getOrElse(false), isSpam = metadata.isSpam.getOrElse(false), isSoftNsfw = metadata.isSoftNsfw.getOrElse(false), isLowQuality = metadata.isHighQuality.exists(!_), isOcr = metadata.isOcr.getOrElse(false), ) } val categoryScoreMap = data.annotations.entities // This gives Option[List[EntityWithMetadata]] .map( _.map(entity => entity.qualifiedId._2.toString -> entity.score .getOrElse(0.0) // Convert each entity to (String, Double) ).toMap ) // Convert List[(String, Double)] to Map[String, Double] hmt.GrokAnnotations( topics = data.topics, tags = data.annotations.tags.getOrElse(Seq.empty).map(_.tag), metadata = metadata, categoryScores = categoryScoreMap, sunnyScore = data.annotations.sunnyScore, politicalInclination = data.annotations.politicalInclination.flatMap { inclination => scala.util .Try(hmt.PoliticalInclination.valueOf(inclination.name)).toOption.flatten }, slopScore = data.annotations.slopScore, ) case _ => throw new Exception("Unknown type returned") } } }.handle { case _ => Seq.fill(tweetIdsToHydrate.size)(None) } } def getFeatureMaps( candidates: Seq[CandidateWithFeatures[TweetCandidate]], ): Future[Seq[FeatureMap]] = { val tweetIdsToHydrate = candidates.map(CandidatesUtil.getOriginalTweetId) val debugStringFeatures = candidates.map(_.features.getOrElse(DebugStringFeature, None).getOrElse("")) val responseMap = getGrokAnnotationsFromHMF(tweetIdsToHydrate) responseMap.map { result => result.zip(debugStringFeatures).map { case (annotations, debugStringFeature) => val categoryScoreMap: Option[Map[String, Double]] = annotations.flatMap(_.categoryScores.map(_.toMap)) val tags: Set[String] = annotations.map(_.tags).getOrElse(Seq.empty).toSet val grokSlopFeature = annotations.flatMap(_.slopScore.map(_.toLong)) FeatureMapBuilder() .add(GrokAnnotationsFeature, annotations) .add(GrokCategoryDataRecordFeature, categoryScoreMap) .add( GrokTopCategoryFeature, annotations .flatMap(_.categoryScores) .flatMap { scores => val validCategories = scores.collect { case (category, score) if category.forall(_.isDigit) && GrokCategoryIdToNameMap.contains(category.toLong) => (category.toLong, score) } if (validCategories.nonEmpty) { Some(validCategories.maxBy(_._2)._1) } else { None } } ) .add(GrokTagsFeature, tags.map(_.toLowerCase)) .add(GrokIsGoreFeature, annotations.flatMap(_.metadata.map(_.isGore))) .add(GrokIsNsfwFeature, annotations.flatMap(_.metadata.map(_.isNsfw))) .add(GrokIsSpamFeature, annotations.flatMap(_.metadata.map(_.isSpam))) .add(GrokIsViolentFeature, annotations.flatMap(_.metadata.map(_.isViolent))) .add(GrokIsLowQualityFeature, annotations.flatMap(_.metadata.map(_.isLowQuality))) .add(GrokIsOcrFeature, annotations.flatMap(_.metadata.map(_.isOcr))) .add(GrokSunnyScoreFeature, annotations.flatMap(_.sunnyScore)) // Used only for metrics tracking. Does not affect the recommendations. .add(GrokPoliticalInclinationFeature, annotations.flatMap(_.politicalInclination)) .add(GrokSlopScoreFeature, grokSlopFeature) .add( DebugStringFeature, Some("%s GrokSlop:%s" .format(debugStringFeature, grokSlopFeature.map(_.toString).getOrElse("None")))) .build() case _ => FeatureMapBuilder() .add(GrokAnnotationsFeature, None) .add(GrokCategoryDataRecordFeature, None) .add(GrokTopCategoryFeature, None) .add(GrokTagsFeature, Set.empty[String]) .add(GrokIsGoreFeature, None) .add(GrokIsNsfwFeature, None) .add(GrokIsSpamFeature, None) .add(GrokIsViolentFeature, None) .add(GrokIsLowQualityFeature, None) .add(GrokIsOcrFeature, None) .add(GrokSunnyScoreFeature, None) .add(GrokPoliticalInclinationFeature, None) .add(GrokSlopScoreFeature, None) .add(DebugStringFeature, None) .build() } } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { OffloadFuturePools.offloadBatchSeqToFutureSeq( candidates, getFeatureMaps, batchSize ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GrokGorkContentCreatorFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.GrokContentCreatorFeature import com.twitter.home_mixer.model.HomeFeatures.GorkContentCreatorFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch /** * Used only for metrics tracking to measure how often we are serving these posts */ object GrokGorkContentCreatorFeatureHydrator extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GrokGorkContentCreator") override val features: Set[Feature[_, _]] = Set(GrokContentCreatorFeature, GorkContentCreatorFeature) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { candidates.map { candidate => val authorIdOpt = candidate.features.getOrElse(AuthorIdFeature, None) FeatureMap( GrokContentCreatorFeature, authorIdOpt.contains(GrokCreatorId), GorkContentCreatorFeature, authorIdOpt.contains(GorkCreatorId), ) } } private val GrokCreatorId = // @grok private val GorkCreatorId = // @gork } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GrokTranslatedPostIsCachedFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.GrokTranslatedPostIsCachedFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableGrokAutoTranslateLanguageFilter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.strato.generated.client.searchai.grok.TranslatedPostClientColumn import com.twitter.search_router.thriftscala.GrokTranslateData import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton @Singleton class GrokTranslatedPostIsCachedFeatureHydrator @Inject() ( translatedPostClientColumn: TranslatedPostClientColumn, statsReceiver: StatsReceiver) extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] with Logging { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GrokTranslatedPostIsCached") private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val successCounter = scopedStatsReceiver.counter("cacheFetch/success") private val failedCounter = scopedStatsReceiver.counter("cacheFetch/failure") private val cacheHitCounter = scopedStatsReceiver.counter("cacheHit") private val cacheMissCounter = scopedStatsReceiver.counter("cacheMiss") private val DefaultFeatureMap = FeatureMapBuilder().add(GrokTranslatedPostIsCachedFeature, true).build() private val fetcher: Fetcher[(Long, String), Unit, GrokTranslateData] = translatedPostClientColumn.fetcher override val features: Set[Feature[_, _]] = Set(GrokTranslatedPostIsCachedFeature) override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableGrokAutoTranslateLanguageFilter) override def apply( query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap ): Stitch[FeatureMap] = { val tweetId = candidate.id val dstLangOpt = query.getLanguageCode dstLangOpt match { case Some(dstLang) => fetcher .fetch((tweetId, dstLang), ()).map { result => successCounter.incr() val cacheAvailable = result.v.isDefined if (cacheAvailable) cacheHitCounter.incr() else cacheMissCounter.incr() FeatureMapBuilder() .add(GrokTranslatedPostIsCachedFeature, cacheAvailable) .build() }.rescue { case _ => failedCounter.incr() Stitch.value(DefaultFeatureMap) } case None => Stitch.value(DefaultFeatureMap) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/HeartbeatOptimizerParamsHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.Counter import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.HeartbeatOptimizerParamsMHPkey import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.strato.columns.heartbeat_optimizer.thriftscala.OptimizerMHParamsValue import com.twitter.strato.columns.heartbeat_optimizer.thriftscala.ParameterAndValue import com.twitter.strato.generated.client.heartbeat_optimizer.HeartbeatOptimizerParamsMHClientColumn import javax.inject.Inject import javax.inject.Singleton object HeartbeatOptimizerWeightsFeature extends Feature[PipelineQuery, Option[OptimizerParams]] case class OptimizerParams( epochTimestamp: Long, nBuckets: Int, optimizerWeights: Seq[Map[String, Double]]) @Singleton class HeartbeatOptimizerParamsHydrator @Inject() ( heartbeatOptimizerParamsMHClientColumn: HeartbeatOptimizerParamsMHClientColumn, statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "HeartbeatOptimizerParams") private val fetcher: Fetcher[(String, String), Unit, OptimizerMHParamsValue] = heartbeatOptimizerParamsMHClientColumn.fetcher override val features: Set[Feature[_, _]] = Set(HeartbeatOptimizerWeightsFeature) private val LKEY = "0" private val sucessStat: Counter = statsReceiver.counter("success") private val failureStat: Counter = statsReceiver.counter("failure") override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val pkey = query.params(HeartbeatOptimizerParamsMHPkey) fetcher .fetch((pkey, LKEY)).map { manhattanResult => manhattanResult.v match { case Some(optimizerMHParamsValue) => val parameterAndValueList: Seq[Seq[ParameterAndValue]] = optimizerMHParamsValue.parameterAndValueList val paramMaps: Seq[Map[String, Double]] = parameterAndValueList.map { paramAndValueList => paramAndValueList.map { paramAndValue => (paramAndValue.parameter, paramAndValue.value) }.toMap } val optimizerWeights = OptimizerParams( epochTimestamp = optimizerMHParamsValue.epochTimestamp.toLong, nBuckets = paramMaps.size, optimizerWeights = paramMaps ) sucessStat.incr(1) FeatureMapBuilder() .add(HeartbeatOptimizerWeightsFeature, Some(optimizerWeights)) .build() case _ => failureStat.incr(1) FeatureMapBuilder() .add(HeartbeatOptimizerWeightsFeature, None) .build() } }.handle { case _ => failureStat.incr(1) FeatureMapBuilder() .add(HeartbeatOptimizerWeightsFeature, None) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/HeavyRankerWeightsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import breeze.stats.distributions.Beta import breeze.stats.distributions.Rand import com.github.nscala_time.time.Imports.LocalDate import com.twitter.home_mixer.model.PredictedScoreFeature import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.AddNoiseInWeightsPerLabel import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.EnableDailyFrozenNoisyWeights import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.NoisyWeightAlphaParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.NoisyWeightBetaParam import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import java.lang.{Long => JLong} import java.util.Objects import javax.inject.Inject import javax.inject.Singleton import com.twitter.ml.api.thriftscala.GeneralTensor import com.twitter.ml.api.thriftscala.DoubleTensor @Singleton class HeavyRankerWeightsQueryFeatureHydrator @Inject() () extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("HeavyRankerWeights") private def doubleToGeneralTensor(value: Double): GeneralTensor = { val tensor = DoubleTensor(doubles = List(value), shape = Some(List(1L))) GeneralTensor.DoubleTensor(tensor) } override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val builder = FeatureMapBuilder() // Ensure that for the same userId and start of day, we get the same sequence of random numbers // Let's us freeze the weight for one day and observe UAS impact val startOfDay = LocalDate.now().toDateTimeAtStartOfDay.toInstant.getMillis val seed = Objects.hash(JLong.valueOf(query.getRequiredUserId), JLong.valueOf(startOfDay)).toLong val alpha = query.params(NoisyWeightAlphaParam) val beta = query.params(NoisyWeightBetaParam) val betaDist = new Beta(alpha, beta) if (query.params(EnableDailyFrozenNoisyWeights)) Rand.generator.setSeed(seed) for (predictedScoreFeature <- PredictedScoreFeature.PredictedScoreFeatures) { val presetWeight = query.params(predictedScoreFeature.modelWeightParam) val weight = if (query.params(AddNoiseInWeightsPerLabel)) { // Apply noise to each weight presetWeight * (1 + betaDist.draw()) } else { presetWeight } builder.add(predictedScoreFeature.weightQueryFeature, Some(weight)) for (biasQueryFeature <- predictedScoreFeature.biasQueryFeature; modelBiasParam <- predictedScoreFeature.modelBiasParam) { builder.add(biasQueryFeature, Some(query.params(modelBiasParam))) } for (debiasQueryFeature <- predictedScoreFeature.debiasQueryFeature; modelDebiasParam <- predictedScoreFeature.modelDebiasParam) { val debiasValue = query.params(modelDebiasParam) builder.add(debiasQueryFeature, Some(doubleToGeneralTensor((debiasValue)))) } } Stitch.value(builder.build()) } override val features: Set[Feature[_, _]] = { val weightFeatures = PredictedScoreFeature.PredictedScoreFeatureSet.map(_.weightQueryFeature) val biasFeatures = PredictedScoreFeature.PredictedScoreFeatureSet.flatMap(_.biasQueryFeature) val debiasFeatures = PredictedScoreFeature.PredictedScoreFeatureSet .flatMap(_.debiasQueryFeature) (weightFeatures ++ biasFeatures ++ debiasFeatures).toSet } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressedImageClusterIdsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.ImageClipClusterIdInMemCache import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.InProcessCache import com.twitter.stitch.Stitch import com.twitter.storehaus.ReadableStore import com.twitter.timelines.impressionstore.thriftscala.ImpressionList import com.twitter.strato.client.Client import com.twitter.strato.client.Fetcher import com.twitter.util.Future import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton case object ImpressedImageClusterIds extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { override val defaultValue: Seq[Long] = Seq.empty } /** * Get the list of image cluster ids that the user has already seen. */ @Singleton case class ImpressedImageClusterIdsQueryFeatureHydrator @Inject() ( stratoClient: Client, tweetImpressionStore: ReadableStore[Long, ImpressionList], @Named(ImageClipClusterIdInMemCache) imageClipClusterIdInMemCache: InProcessCache[ Long, Option[Option[Long]] ], statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "ImpressedImageClusterIdsQuery") override val features: Set[Feature[_, _]] = Set(ImpressedImageClusterIds) val clusterIdFetcher: Fetcher[Long, Unit, Long] = stratoClient.fetcher[Long, Unit, Long]( "videoRecommendations/twitterClip/twitterClipImageClusterIdMh95") private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("impressionKey/found") private val keyNotFoundCounter = scopedStatsReceiver.counter("impressionKey/notFound") private val cacheHitCounter = scopedStatsReceiver.counter("cache/hit") private val cacheMissCounter = scopedStatsReceiver.counter("cache/miss") private val fetchExceptionCounter = scopedStatsReceiver.counter("getFromCacheOrFetch/exception") private def getImageClipClusterId(tweetId: Long): Stitch[Option[Option[Long]]] = { imageClipClusterIdInMemCache .get(tweetId) .map { cachedValue => cacheHitCounter.incr() Stitch.value(cachedValue) }.getOrElse { cacheMissCounter.incr() clusterIdFetcher .fetch(tweetId) .map(_.v) .liftToOption() .flatMap { clusterIdOpt => imageClipClusterIdInMemCache.set(tweetId, clusterIdOpt) Stitch.value(clusterIdOpt) } }.handle { case _: Exception => fetchExceptionCounter.incr() None } } override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { query.getOptionalUserId match { case Some(userId) => val featureMapResult: Future[Seq[Long]] = tweetImpressionStore .get(userId).map { impressionListOpt => if (impressionListOpt.isEmpty) { keyNotFoundCounter.incr() } keyFoundCounter.incr() val tweetIdsOpt = for { impressionList <- impressionListOpt impressions <- impressionList.impressions } yield { impressions.take(250).map(_.tweetId) } tweetIdsOpt.getOrElse(Seq.empty) } Stitch.callFuture(featureMapResult).flatMap { tweetIds => Stitch .traverse(tweetIds) { tweetId => getImageClipClusterId(tweetId) }.map { results: Seq[Option[Option[Long]]] => FeatureMapBuilder().add(ImpressedImageClusterIds, results.flatten.flatten).build() } } case None => val featureMapResult = FeatureMapBuilder().add(ImpressedImageClusterIds, Seq.empty).build() Stitch.value(featureMapResult) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressedMediaClusterIdsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClusterId95Store import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClipClusterIdInMemCache import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.InProcessCache import com.twitter.stitch.Stitch import com.twitter.storehaus.ReadableStore import com.twitter.timelines.impressionstore.thriftscala.ImpressionList import com.twitter.util.Future import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton case object ImpressedMediaClusterIds extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { override val defaultValue: Seq[Long] = Seq.empty } /** * Get the list of media cluster ids that the user has already seen. */ @Singleton case class ImpressedMediaClusterIdsQueryFeatureHydrator @Inject() ( @Named(MediaClusterId95Store) clusterIdStore: ReadableStore[Long, Long], tweetImpressionStore: ReadableStore[Long, ImpressionList], @Named(MediaClipClusterIdInMemCache) mediaClipClusterIdInMemCache: InProcessCache[ Long, Option[Option[Long]] ], statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "ImpressedMediaClusterIdsQuery") override val features: Set[Feature[_, _]] = Set(ImpressedMediaClusterIds) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("impressionKey/found") private val keyNotFoundCounter = scopedStatsReceiver.counter("impressionKey/notFound") private val cacheHitCounter = scopedStatsReceiver.counter("cache/hit") private val cacheMissCounter = scopedStatsReceiver.counter("cache/miss") private val storeHitCounter = scopedStatsReceiver.counter("store/hit") private val storeMissCounter = scopedStatsReceiver.counter("store/miss") private val fetchExceptionCounter = scopedStatsReceiver.counter("fetch/exception") private def getmediaClipClusterId(tweetId: Long): Stitch[Option[Option[Long]]] = { mediaClipClusterIdInMemCache .get(tweetId) .map { cachedValue => cacheHitCounter.incr() Stitch.value(cachedValue) }.getOrElse { cacheMissCounter.incr() Stitch .callFuture(clusterIdStore.get(tweetId)) .flatMap { clusterIdOpt => if (clusterIdOpt.isDefined) { storeHitCounter.incr() } else { storeMissCounter.incr() } val wrappedClusterIdOpt = Some(clusterIdOpt) mediaClipClusterIdInMemCache.set(tweetId, wrappedClusterIdOpt) Stitch.value(wrappedClusterIdOpt) } }.handle { case _: Exception => fetchExceptionCounter.incr() None } } override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { query.getOptionalUserId match { case Some(userId) => val featureMapResult: Future[Seq[Long]] = tweetImpressionStore .get(userId).map { impressionListOpt => if (impressionListOpt.isEmpty) { keyNotFoundCounter.incr() } keyFoundCounter.incr() val tweetIdsOpt = for { impressionList <- impressionListOpt impressions <- impressionList.impressions } yield { impressions.take(250).map(_.tweetId) } tweetIdsOpt.getOrElse(Seq.empty) } Stitch.callFuture(featureMapResult).flatMap { tweetIds => Stitch .traverse(tweetIds) { tweetId => getmediaClipClusterId(tweetId) }.map { results: Seq[Option[Option[Long]]] => FeatureMapBuilder().add(ImpressedMediaClusterIds, results.flatten.flatten).build() } } case None => val featureMapResult = FeatureMapBuilder().add(ImpressedMediaClusterIds, Seq.empty).build() Stitch.value(featureMapResult) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.MemcachedImpressionBloomFilterStore import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.storehaus.Store import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton case class ImpressionBloomFilterQueryFeatureHydrator @Inject() ( @Named(MemcachedImpressionBloomFilterStore) bloomFilterClient: Store[ blm.ImpressionBloomFilterKey, blm.ImpressionBloomFilterSeq ]) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ImpressionBloomFilter") override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature) private val SurfaceArea = blm.SurfaceArea.HomeTimeline override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId Stitch.callFuture { bloomFilterClient .get(blm.ImpressionBloomFilterKey(userId, SurfaceArea)) .map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty))) .map { bloomFilterSeq => FeatureMapBuilder().add(ImpressionBloomFilterFeature, bloomFilterSeq).build() } } } override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object InNetworkFeatureHydrator extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("InNetwork") override val features: Set[Feature[_, _]] = Set(InNetworkFeature) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { val viewerId = query.getRequiredUserId val followedUserIds = query.features.get.get(SGSFollowedUsersFeature).toSet val featureMaps = candidates.map { candidate => // We use authorId and not sourceAuthorId here so that retweets are defined as in network val isInNetworkOpt = candidate.features.getOrElse(AuthorIdFeature, None).map { authorId => // Users cannot follow themselves but this is in network by definition val isSelfTweet = authorId == viewerId isSelfTweet || followedUserIds.contains(authorId) } FeatureMapBuilder().add(InNetworkFeature, isInNetworkOpt.getOrElse(true)).build() } Stitch.value(featureMaps) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/LastNegativeFeedbackTimeQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.LastNegativeFeedbackTimeFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.recommendations.user_signal_service.SignalsClientColumn import com.twitter.usersignalservice.thriftscala.BatchSignalRequest import com.twitter.usersignalservice.thriftscala.ClientIdentifier import com.twitter.usersignalservice.thriftscala.SignalRequest import com.twitter.usersignalservice.thriftscala.SignalType import com.twitter.usersignalservice.thriftscala.{Signal => UssSignal} import com.twitter.util.Time import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton @Singleton class LastNegativeFeedbackTimeQueryFeatureHydrator @Inject() ( signalsClientColumn: SignalsClientColumn) extends QueryFeatureHydrator[PipelineQuery] with Logging { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("USSSignals") private val fetcher = signalsClientColumn.fetcher override val features: Set[Feature[_, _]] = Set( LastNegativeFeedbackTimeFeature, ) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val lastNegativeFeedbackTime = getLastNegativeFeedbackTime(query.getRequiredUserId) lastNegativeFeedbackTime.map { FeatureMap(LastNegativeFeedbackTimeFeature, _) } } def getLastNegativeFeedbackTime(userId: Long): Stitch[Option[Time]] = { val enabledNegativeSignalTypes = Seq( SignalType.AccountBlock, SignalType.AccountMute, SignalType.TweetSeeFewer, SignalType.TweetReport, SignalType.TweetDontLike) // negative signals val maybeNegativeSignals = enabledNegativeSignalTypes.map { negativeSignal => SignalRequest( maxResults = Some(1), // Only most recent needed signalType = negativeSignal ) } val batchSignalRequest = BatchSignalRequest(userId, maybeNegativeSignals, Some(ClientIdentifier.CrMixerHome)) val signalsStitch = fetcher .fetch(batchSignalRequest) .map { result => result.v .map(_.signalResponse.toSeq.flatMap { case (_, signals) => signals }).getOrElse(Seq.empty) } val getTimestamp: UssSignal => Option[Time] = signal => if (signal.timestamp == 0L) None else Some(Time.fromMilliseconds(signal.timestamp)) signalsStitch.map { _.map(getTimestamp).flatten.reduceOption((a, b) => if (a < b) a else b) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/LastNonPollingTimeQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.FollowingLastNonPollingTimeFeature import com.twitter.home_mixer.model.HomeFeatures.LastNonPollingTimeFeature import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.user_session_store.ReadRequest import com.twitter.user_session_store.ReadWriteUserSessionStore import com.twitter.user_session_store.UserSessionDataset import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton @Singleton case class LastNonPollingTimeQueryFeatureHydrator @Inject() ( userSessionStore: ReadWriteUserSessionStore) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("LastNonPollingTime") override val features: Set[Feature[_, _]] = Set( FollowingLastNonPollingTimeFeature, LastNonPollingTimeFeature, NonPollingTimesFeature ) private val datasets: Set[UserSessionDataset] = Set(UserSessionDataset.NonPollingTimes) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { userSessionStore .read(ReadRequest(query.getRequiredUserId, datasets)) .map { userSession => val nonPollingTimestamps = userSession.flatMap(_.nonPollingTimestamps) val lastNonPollingTime = nonPollingTimestamps .flatMap(_.nonPollingTimestampsMs.headOption) .map(Time.fromMilliseconds) val followingLastNonPollingTime = nonPollingTimestamps .flatMap(_.mostRecentHomeLatestNonPollingTimestampMs) .map(Time.fromMilliseconds) val nonPollingTimes = nonPollingTimestamps .map(_.nonPollingTimestampsMs) .getOrElse(Seq.empty) FeatureMapBuilder() .add(FollowingLastNonPollingTimeFeature, followingLastNonPollingTime) .add(LastNonPollingTimeFeature, lastNonPollingTime) .add(NonPollingTimesFeature, nonPollingTimes) .build() } } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ListIdsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.param.HomeGlobalParams.ListMandarinTweetsParams.ListMandarinTweetsEnable import com.twitter.home_mixer.param.HomeGlobalParams.ListMandarinTweetsParams.ListMandarinTweetsLists import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.socialgraph.{thriftscala => sg} import com.twitter.stitch.Stitch import com.twitter.stitch.socialgraph.SocialGraph import javax.inject.Inject import javax.inject.Singleton case object ListIdsFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { override val defaultValue: Seq[Long] = Seq.empty } @Singleton class ListIdsQueryFeatureHydrator @Inject() (socialGraph: SocialGraph) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ListIds") override val features: Set[Feature[_, _]] = Set(ListIdsFeature) private val MaxListsToFetch = 20 private def buildIdsRequest(userId: Long, relationshipType: sg.RelationshipType) = sg.IdsRequest( relationships = Seq( sg.SrcRelationship(userId, relationshipType, hasRelationship = true), sg.SrcRelationship(userId, sg.RelationshipType.ListMuting, hasRelationship = false) ), pageRequest = Some(sg.PageRequest(selectAll = Some(false), count = Some(MaxListsToFetch))), context = Some(sg.LookupContext(performUnion = Some(false), includeAll = Some(false))) ) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId val subscribedRequest = buildIdsRequest(userId, sg.RelationshipType.ListIsSubscriber) val ownedRequest = buildIdsRequest(userId, sg.RelationshipType.ListOwning) Stitch.join(socialGraph.ids(ownedRequest), socialGraph.ids(subscribedRequest)).map { case (ownedResponse, subscribedResponse) => val recommendedListIds = if (query.params(ListMandarinTweetsEnable)) query.params(ListMandarinTweetsLists) else Seq.empty val ids = (ownedResponse.ids ++ subscribedResponse.ids ++ recommendedListIds).distinct FeatureMapBuilder().add(ListIdsFeature, ids).build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MediaClusterIdFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.MediaCategoryFeature import com.twitter.home_mixer.model.HomeFeatures.TweetMediaClusterIdsFeature import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClipClusterIdInMemCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClusterId95Store import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableMediaClusterFeatureHydrationParam import com.twitter.mediaservices.commons.thriftscala.MediaCategory import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.cache.InProcessCache import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import com.twitter.storehaus.ReadableStore @Singleton class MediaClusterIdFeatureHydrator @Inject() ( @Named(MediaClusterId95Store) clusterIdStore: ReadableStore[Long, Long], @Named(MediaClipClusterIdInMemCache) mediaClipClusterIdInMemCache: InProcessCache[ Long, Option[Option[Long]] ], statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] with WithDefaultFeatureMap { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("MediaClusterId") override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableMediaClusterFeatureHydrationParam) override val features: Set[Feature[_, _]] = Set(TweetMediaClusterIdsFeature) override val defaultFeatureMap: FeatureMap = FeatureMap(TweetMediaClusterIdsFeature, Map.empty[Long, Long]) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") private val cacheHitCounter = scopedStatsReceiver.counter("cache/hit") private val cacheMissCounter = scopedStatsReceiver.counter("cache/miss") private val fetchExceptionCounter = scopedStatsReceiver.counter("getFromCacheOrFetch/exception") private def getFromCacheOrFetch(tweetId: Long): Stitch[Option[Option[Long]]] = { mediaClipClusterIdInMemCache .get(tweetId) .map { cachedValue => cacheHitCounter.incr() Stitch.value(cachedValue) }.getOrElse { cacheMissCounter.incr() // Use the store which handles memcache + Manhattan fallback Stitch .callFuture(clusterIdStore.get(tweetId)).map { result => result match { case Some(clusterId) => keyFoundCounter.incr() val finalResult = Some(Some(clusterId)) mediaClipClusterIdInMemCache.set(tweetId, finalResult) finalResult case None => keyNotFoundCounter.incr() mediaClipClusterIdInMemCache.set(tweetId, Some(None)) Some(None) } }.handle { case ex: Exception => fetchExceptionCounter.incr() ex.printStackTrace() None } } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { Stitch.collect( candidates.map { candidate => val mediaIds = candidate.features.getOrElse(TweetMediaIdsFeature, Seq.empty[Long]) val mediaCategory = candidate.features.getOrElse(MediaCategoryFeature, None) if (mediaCategory.contains(MediaCategory.TweetVideo) || mediaCategory.contains( MediaCategory.AmplifyVideo)) { val items: Seq[Stitch[Option[(Long, Long)]]] = mediaIds.map { mediaId => getFromCacheOrFetch(candidate.candidate.id).map { case Some(Some(mediaClusterId)) => keyFoundCounter.incr() Some(mediaId -> mediaClusterId) case _ => keyNotFoundCounter.incr() None } } Stitch.collect(items).map { results => val mediaClusterMap = results.flatten.toMap // Flatten to remove None and create Map[Long, Long] FeatureMapBuilder().add(TweetMediaClusterIdsFeature, mediaClusterMap).build() } } else { Stitch.value( FeatureMapBuilder().add(TweetMediaClusterIdsFeature, Map.empty[Long, Long]).build()) } } ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MediaCompletionRateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.TweetMediaCompletionRateFeature import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaCompletionRateInMemCache import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.cache.InProcessCache import com.twitter.stitch.Stitch import com.twitter.strato.client.Client import com.twitter.strato.generated.client.analytics.video.VideoCompletionRateOnApiMediaClientColumn import com.twitter.strato.graphql.thriftscala.ApiMediaKey.TweetVideo import com.twitter.strato.graphql.thriftscala.TweetVideoKey import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class MediaCompletionRateFeatureHydrator @Inject() ( @Named(BatchedStratoClientWithModerateTimeout) stratoClient: Client, @Named(MediaCompletionRateInMemCache) mediaCompletionRateInMemCache: InProcessCache[Long, Double], statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("MediaCompletionRate") override val features: Set[Feature[_, _]] = Set(TweetMediaCompletionRateFeature) private val completionRateFetcher = stratoClient.fetcher[ VideoCompletionRateOnApiMediaClientColumn.Key, VideoCompletionRateOnApiMediaClientColumn.View, VideoCompletionRateOnApiMediaClientColumn.Value ](VideoCompletionRateOnApiMediaClientColumn.Path) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") private val cacheHitCounter = scopedStatsReceiver.counter("cache/hit") private val cacheMissCounter = scopedStatsReceiver.counter("cache/miss") private val fetchExceptionCounter = scopedStatsReceiver.counter("getFromCacheOrFetch/exception") private def getFromCacheOrFetch(mediaId: Long): Stitch[Option[Double]] = { mediaCompletionRateInMemCache .get(mediaId) .map { cachedValue => cacheHitCounter.incr() Stitch.value(Some(cachedValue)) }.getOrElse { cacheMissCounter.incr() val key = TweetVideo(TweetVideoKey(mediaId = mediaId, tweetId = 0L)) completionRateFetcher .fetch(key) .flatMap { result => val completionRate = result.v.map(_.organic) mediaCompletionRateInMemCache.set(mediaId, completionRate.getOrElse(0.0)) Stitch.value(completionRate) } }.handle { case _: Exception => fetchExceptionCounter.incr() None } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { val featureMaps = candidates.map { candidate => val hasVideo = candidate.features.getOrElse(HasVideoFeature, false) val mediaIds = candidate.features.getOrElse(TweetMediaIdsFeature, Seq.empty[Long]) val inNetwork = candidate.features.getOrElse(FromInNetworkSourceFeature, false) val completionRates = if (hasVideo && !inNetwork) { mediaIds.map { mediaId => getFromCacheOrFetch(mediaId).map { case Some(completionRate) => keyFoundCounter.incr() Some(completionRate) case _ => keyNotFoundCounter.incr() None } } } else Seq.empty[Stitch[Option[Double]]] Stitch.collectToTry(completionRates).map { results => val completionRates = results.map(_.toOption.flatten).flatten val maxCompletionRate = if (completionRates.nonEmpty) Some(completionRates.max) else None FeatureMap(TweetMediaCompletionRateFeature, maxCompletionRate) } } Stitch.collect(featureMaps) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/MultiModalEmbeddingsFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.MultiModalEmbeddingsFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.cache.ExpiringLruInProcessCache import com.twitter.servo.cache.InProcessCache import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.searchai.storage.PostMultimodalEmbeddingMhClientColumn import com.twitter.util.Duration import javax.inject.Inject import javax.inject.Singleton import scala.util.Random import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.util.Try object MultiModalEmbeddingsFeatureHydrator { private val BaseTTL = 15 private val TTL: Duration = (BaseTTL + Random.nextInt(10)).minutes private val maximumCacheSize = 15000 // 15000 * 1536 * 8 bytes = <200Mb val cache: InProcessCache[Long, Option[Option[Seq[Double]]]] = new ExpiringLruInProcessCache(ttl = TTL, maximumSize = maximumCacheSize) private val embeddingFetcherModelVersion = "v1" } @Singleton class MultiModalEmbeddingsFeatureHydrator @Inject() ( postMultimodalEmbeddingMhClientColumn: PostMultimodalEmbeddingMhClientColumn, statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("MultiModalEmbeddings") override val features: Set[Feature[_, _]] = Set(MultiModalEmbeddingsFeature) private val embeddingFetcher = postMultimodalEmbeddingMhClientColumn.fetcher private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") private val fetchExceptionCounter = scopedStatsReceiver.counter("getFromCacheOrFetch/exception") private def getEmbedding( tweetId: Long ): Stitch[Option[Option[Seq[Double]]]] = { val embeddingStitchFromCache: Option[Stitch[Option[Option[Seq[Double]]]]] = MultiModalEmbeddingsFeatureHydrator.cache .get(tweetId) .map(Stitch.value) embeddingStitchFromCache match { case Some(stitch: Stitch[Option[Option[Seq[Double]]]]) => stitch case None => val embeddingStitch = embeddingFetcher .fetch((tweetId, MultiModalEmbeddingsFeatureHydrator.embeddingFetcherModelVersion)).map { result => result.v match { case Some(tweetEmbedding) if tweetEmbedding != null && tweetEmbedding.embedding1 != null && tweetEmbedding.embedding1.isDefined => keyFoundCounter.incr() val embedding = tweetEmbedding.embedding1 MultiModalEmbeddingsFeatureHydrator.cache.set(tweetId, Some(embedding)) Some(embedding) case _ => keyNotFoundCounter.incr() MultiModalEmbeddingsFeatureHydrator.cache.set(tweetId, Some(None)) None } }.handle { case ex: Exception => fetchExceptionCounter.incr() None } embeddingStitch } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { val stitchesWithTry: Stitch[Seq[Try[Option[Option[Seq[Double]]]]]] = Stitch.collectToTry { candidates.map { candidate => val tweetId: Long = candidate.candidate.id getEmbedding(tweetId) } } stitchesWithTry.map { tryVal => tryVal.flatMap(_.toOption).map { case Some(embedding) => FeatureMap(MultiModalEmbeddingsFeature, embedding) case _ => FeatureMap(MultiModalEmbeddingsFeature, None) } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NamesFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.gizmoduck.{thriftscala => gt} import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.stitch.gizmoduck.Gizmoduck import com.twitter.util.Return import javax.inject.Inject import javax.inject.Singleton protected case class ProfileNames(screenName: String, realName: String) @Singleton class NamesFeatureHydrator @Inject() (gizmoduck: Gizmoduck) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Names") override val features: Set[Feature[_, _]] = Set(ScreenNamesFeature, RealNamesFeature) override def onlyIf(query: PipelineQuery): Boolean = query.product match { case FollowingProduct => false case _ => true } private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Profile) /** * The UI currently only ever displays the first 2 names in social context lines * E.g. "User and 3 others like" or "UserA and UserB liked" */ private val MaxCountUsers = 2 override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { val candidateUserIdsMap = candidates.map { candidate => candidate.candidate.id -> (candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++ candidate.features.getOrElse(FollowedByUserIdsFeature, Nil).take(MaxCountUsers) ++ candidate.features.getOrElse(AuthorIdFeature, None) ++ candidate.features.getOrElse(SourceUserIdFeature, None)).distinct }.toMap val distinctUserIds = candidateUserIdsMap.values.flatten.toSeq.distinct Stitch .collectToTry(distinctUserIds.map(userId => gizmoduck.getUserById(userId, queryFields))) .map { allUsers => val idToProfileNamesMap = allUsers.flatMap { case Return(allUser) => allUser.profile .map(profile => allUser.id -> ProfileNames(profile.screenName, profile.name)) case _ => None }.toMap val validUserIds = idToProfileNamesMap.keySet candidates.map { candidate => val combinedMap = candidateUserIdsMap .getOrElse(candidate.candidate.id, Nil) .flatMap { case userId if validUserIds.contains(userId) => idToProfileNamesMap.get(userId).map(profileNames => userId -> profileNames) case _ => None } val perCandidateRealNameMap = combinedMap.map { case (k, v) => k -> v.realName }.toMap val perCandidateScreenNameMap = combinedMap.map { case (k, v) => k -> v.screenName }.toMap FeatureMapBuilder() .add(ScreenNamesFeature, perCandidateScreenNameMap) .add(RealNamesFeature, perCandidateRealNameMap) .build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NaviClientConfigQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.NaviClientConfigFeature import com.twitter.home_mixer.model.NaviClientConfig import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelIdParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.NaviGPUBatchSizeParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ProdModelIdParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.UseGPUNaviClusterParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.UseGPUNaviClusterTestUsersParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.UseRealtimeNaviClusterParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.UseSecondaryNaviClusterParam import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecap import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapGPU import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapRealtime import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapSecondary import com.twitter.product_mixer.component_library.module.TestUserMapper import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class NaviClientConfigQueryFeatureHydrator @Inject() ( testUserMapper: TestUserMapper) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("NaviClientConfig") override val features: Set[Feature[_, _]] = Set(NaviClientConfigFeature) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { // Use experimental realtime cluster val useRealtimeCluster = query.params(UseRealtimeNaviClusterParam) // Use secondary navi cluster for handling peak traffic val useSecondaryCluster = query.params(UseSecondaryNaviClusterParam) val modelId = query.params(ModelIdParam) val prodModelId = query.params(ProdModelIdParam) val isTestUser = testUserMapper.isTestUser(query.clientContext) val useGPUCluster = prodModelId == modelId && (query.params(UseGPUNaviClusterParam) || (isTestUser && query.params(UseGPUNaviClusterTestUsersParam))) val gpuBatchSize = if (useGPUCluster) Some(query.params(NaviGPUBatchSizeParam).toInt) else None val clusterClientPriorityList = Seq( (useGPUCluster, NaviModelClientHomeRecapGPU), (useRealtimeCluster, NaviModelClientHomeRecapRealtime), (useSecondaryCluster, NaviModelClientHomeRecapSecondary) ) val clientName = clusterClientPriorityList .collectFirst { case (true, name) => name }.getOrElse(NaviModelClientHomeRecap) val clusterStr = clientName match { case NaviModelClientHomeRecapGPU => "GPU" case NaviModelClientHomeRecapRealtime => "Realtime" case _ => "" } val naviConfig = NaviClientConfig(clientName, gpuBatchSize, clusterStr) Stitch.value(FeatureMap(NaviClientConfigFeature, naviConfig)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/NaviVideoClientConfigQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.NaviClientConfigFeature import com.twitter.home_mixer.model.NaviClientConfig import com.twitter.home_mixer.param.HomeGlobalParams.Scoring._ import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapVideo import com.twitter.product_mixer.component_library.module.TestUserMapper import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class NaviVideoClientConfigQueryFeatureHydrator @Inject() ( testUserMapper: TestUserMapper) extends NaviClientConfigQueryFeatureHydrator(testUserMapper) { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("NaviVideoClientConfig") override val features: Set[Feature[_, _]] = Set(NaviClientConfigFeature) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { query.params(UseVideoNaviClusterParam) match { case true => val config = NaviClientConfig( clientName = NaviModelClientHomeRecapVideo, customizedBatchSize = None, clusterStr = "Video" ) Stitch.value(FeatureMap(NaviClientConfigFeature, config)) case _ => super.hydrate(query) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OnPremRealGraphQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableOnPremRealGraphQueryFeatures import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.recommendations.interaction_graph.on_prem.RealGraphUserFeaturesOnUserClientColumn import com.twitter.timelines.real_graph.{thriftscala => rg} import javax.inject.Inject import javax.inject.Singleton @Singleton class OnPremRealGraphQueryFeatureHydrator @Inject() ( realGraphUserFeaturesClientColumn: RealGraphUserFeaturesOnUserClientColumn) extends QueryFeatureHydrator[PipelineQuery] with Conditionally[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("OnPremRealGraphFeatures") override val features: Set[Feature[_, _]] = Set(RealGraphFeatures) override def onlyIf( query: PipelineQuery ): Boolean = { query.params(EnableOnPremRealGraphQueryFeatures) } override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { realGraphUserFeaturesClientColumn.fetcher .fetch(query.getRequiredUserId).map { userSession => val realGraphFeaturesMap = userSession.v.flatMap { userSession => userSession.realGraphFeatures.collect { case rg.RealGraphFeatures.V1(realGraphFeatures) => val edgeFeatures = realGraphFeatures.edgeFeatures ++ realGraphFeatures.oonEdgeFeatures edgeFeatures.map { edge => edge.destId -> edge }.toMap } } FeatureMapBuilder().add(RealGraphFeatures, realGraphFeaturesMap).build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OptimizerWeightsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.PredictedScoreFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import javax.inject.Inject /* * This Hydrator updates the model scoring weights with the optimizer weights. */ class OptimizerWeightsQueryFeatureHydrator @Inject() ( statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("OptimizerWeights") private val EnabledPredictedScoreFeatures: Seq[PredictedScoreFeature] = PredictedScoreFeature.PredictedScoreFeatures override val features: Set[Feature[_, _]] = { EnabledPredictedScoreFeatures.map(_.weightQueryFeature).toSet } private val BIG_PRIME = 65537 private def hashFn(userId: Long, seed: Long, numBuckets: Int): Int = { val x = userId % BIG_PRIME val y = seed % BIG_PRIME val h = (31 * x + 41 * y) * 53 (h % BIG_PRIME).toInt % numBuckets } private val successCounter = statsReceiver.counter("success") private val failureCounter = statsReceiver.counter("failure") private def get_user_weights( userId: Long, optimizerParams: OptimizerParams ): Map[String, Double] = { val weightIndex = hashFn(userId, seed = optimizerParams.epochTimestamp, numBuckets = optimizerParams.nBuckets) val userWeights = optimizerParams.optimizerWeights(weightIndex) userWeights } override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val optimizerParams = query.features .flatMap(_.getOrElse(HeartbeatOptimizerWeightsFeature, None)) val featureMapBuilder = FeatureMapBuilder() optimizerParams match { case Some(optimizerParams) => if (optimizerParams.optimizerWeights.size != optimizerParams.nBuckets) { failureCounter.incr(1) // Fall back to default weights when size mismatch for (predictedScoreFeature <- EnabledPredictedScoreFeatures) { val currentWeight = query.params(predictedScoreFeature.modelWeightParam) featureMapBuilder.add(predictedScoreFeature.weightQueryFeature, Some(currentWeight)) } } else { val userWeights: Map[String, Double] = get_user_weights(query.getRequiredUserId, optimizerParams) for (predictedScoreFeature <- EnabledPredictedScoreFeatures) { val currentWeight = query.params(predictedScoreFeature.modelWeightParam) val weight_name = predictedScoreFeature.modelWeightParam.name val optimizerWeight = userWeights.getOrElse(weight_name, currentWeight) featureMapBuilder.add(predictedScoreFeature.weightQueryFeature, Some(optimizerWeight)) } successCounter.incr(1) } case _ => // When HeartbeatOptimizerWeightsFeature is None, use default weights for (predictedScoreFeature <- EnabledPredictedScoreFeatures) { val currentWeight = query.params(predictedScoreFeature.modelWeightParam) featureMapBuilder.add(predictedScoreFeature.weightQueryFeature, Some(currentWeight)) } failureCounter.incr(1) } Stitch.value(featureMapBuilder.build()) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OriginalAuthorLargeEmbeddingsFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.OriginalAuthorLargeEmbeddingsFeature import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.OriginalAuthorLargeEmbeddingsKeyFeature import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableLargeEmbeddingsFeatureHydrationParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelNameParam import com.twitter.home_mixer.util.CandidatesUtil.getOriginalAuthorId import com.twitter.home_mixer_features.{thriftscala => hmf} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.large_embeddings.HashingFeatureParams import com.twitter.timelines.prediction.adapters.large_embeddings.HomeMixerLargeEmbeddingsFeatureHydrator import com.twitter.timelines.prediction.adapters.large_embeddings.LargeEmbeddingsAdapter import com.twitter.timelines.prediction.adapters.large_embeddings.OriginalAuthorLargeEmbeddingsAdapter import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton @Singleton class OriginalAuthorLargeEmbeddingsFeatureHydrator @Inject() ( statsReceiver: StatsReceiver, override val homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] with HomeMixerLargeEmbeddingsFeatureHydrator { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("OriginalAuthorLargeEmbeddings") override val features: Set[Feature[_, _]] = Set(OriginalAuthorLargeEmbeddingsFeature, OriginalAuthorLargeEmbeddingsKeyFeature) override val adapter: LargeEmbeddingsAdapter = OriginalAuthorLargeEmbeddingsAdapter override val cacheType: hmf.Cache = hmf.Cache.AuthorLargeEmbeddings override val scopedStatsReceiver: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableLargeEmbeddingsFeatureHydrationParam) // Hashing Features override val defaultHashingFeatureParams: HashingFeatureParams = HashingFeatureParams( scales = Seq(3384241453L, 3372414709L), biases = Seq(1649585795L, 3131243219L), modulus = 3957384397L, bucketSize = 3000000L, ) override val modelName2HashingFeatureParams: Map[String, HashingFeatureParams] = Map( "hr_video_prod__v3_realtime" -> HashingFeatureParams( scales = Seq(113294449L, 601841083L), biases = Seq(2231299001L, 841367196L), modulus = 2343760591L, bucketSize = 3000000L, ), "hr_video_prod__v2_lembeds" -> HashingFeatureParams( scales = Seq(787140070L, 633713480L), biases = Seq(427768658L, 911091889L), modulus = 2888480981L, bucketSize = 300000L, ), "hr_prod__v4_embeds_230M" -> HashingFeatureParams( scales = Seq(371965780L, 328930218L), biases = Seq(139686260L, 37755056L), modulus = 631860353L, bucketSize = 30000000L, ), "hr_prod__v5_embeds_230M_and_transformer" -> HashingFeatureParams( scales = Seq(371965780L, 328930218L), biases = Seq(139686260L, 37755056L), modulus = 631860353L, bucketSize = 30000000L, ), "hr_prod__v5_watchtime" -> HashingFeatureParams( scales = Seq(2328530078L, 2844016377L), biases = Seq(1352496802L, 3011003330L), modulus = 3979826519L, bucketSize = 30000000L, ), "hr_prod__v6_transformer_v2" -> HashingFeatureParams( scales = Seq(371965780L, 328930218L), biases = Seq(139686260L, 37755056L), modulus = 631860353L, bucketSize = 30000000L, ), "hr_prod__v6_mixed_training" -> HashingFeatureParams( scales = Seq(371965780L, 328930218L), biases = Seq(139686260L, 37755056L), modulus = 631860353L, bucketSize = 30000000L, ), "hr_prod__v6_transformer_v2_kafka_merge_join" -> HashingFeatureParams( scales = Seq(371965780L, 328930218L), biases = Seq(139686260L, 37755056L), modulus = 631860353L, bucketSize = 30000000L, ), "hr_prod__v6_transformer_v2_realtime_debias_21apr" -> HashingFeatureParams( scales = Seq(371965780L, 328930218L), biases = Seq(139686260L, 37755056L), modulus = 631860353L, bucketSize = 30000000L, ), "hr_video_prod__v4_realtime" -> HashingFeatureParams( scales = Seq(113294449L, 601841083L), biases = Seq(2231299001L, 841367196L), modulus = 2343760591L, bucketSize = 3000000L, ), "hr_video_prod__v4_realtime_mergehead" -> HashingFeatureParams( scales = Seq(113294449L, 601841083L), biases = Seq(2231299001L, 841367196L), modulus = 2343760591L, bucketSize = 3000000L, ), ) private val batchSize = 25 private def getBatchedFeatureMap( modelName: String, candidatesBatch: Seq[CandidateWithFeatures[TweetCandidate]], ): Future[Seq[FeatureMap]] = { val originalAuthorIds = candidatesBatch.map { candidate => getOriginalAuthorId(candidate.features).getOrElse(0L) } getLargeEmbeddings(originalAuthorIds, modelName).map { responses => responses.map { largeEmbeddingResponse => FeatureMapBuilder() .add(OriginalAuthorLargeEmbeddingsFeature, largeEmbeddingResponse.dataRecord) .add(OriginalAuthorLargeEmbeddingsKeyFeature, largeEmbeddingResponse.hashedKeys) .build() } } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val modelName = query.params(ModelNameParam) OffloadFuturePools.offloadBatchSeqToFutureSeq( candidates, getBatchedFeatureMap(modelName, _), batchSize ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/OriginalTweetLargeEmbeddingsFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.OriginalTweetLargeEmbeddingsFeature import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.OriginalTweetLargeEmbeddingsKeyFeature import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableLargeEmbeddingsFeatureHydrationParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelNameParam import com.twitter.home_mixer.util.CandidatesUtil.getOriginalTweetId import com.twitter.home_mixer_features.{thriftscala => hmf} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.large_embeddings.HashingFeatureParams import com.twitter.timelines.prediction.adapters.large_embeddings.HomeMixerLargeEmbeddingsFeatureHydrator import com.twitter.timelines.prediction.adapters.large_embeddings.LargeEmbeddingsAdapter import com.twitter.timelines.prediction.adapters.large_embeddings.OriginalTweetLargeEmbeddingsAdapter import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton @Singleton class OriginalTweetLargeEmbeddingsFeatureHydrator @Inject() ( statsReceiver: StatsReceiver, override val homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] with HomeMixerLargeEmbeddingsFeatureHydrator { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("OriginalTweetLargeEmbeddings") override val features: Set[Feature[_, _]] = Set(OriginalTweetLargeEmbeddingsFeature, OriginalTweetLargeEmbeddingsKeyFeature) override val adapter: LargeEmbeddingsAdapter = OriginalTweetLargeEmbeddingsAdapter override val cacheType: hmf.Cache = hmf.Cache.TweetLargeEmbeddings override val scopedStatsReceiver: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableLargeEmbeddingsFeatureHydrationParam) // Hashing Features override val defaultHashingFeatureParams: HashingFeatureParams = HashingFeatureParams( scales = Seq(1131302000L, 303023026L), biases = Seq(799473858L, 600426834L), modulus = 3588720353L, bucketSize = 10000000L, ) override val modelName2HashingFeatureParams: Map[String, HashingFeatureParams] = Map( "hr_video_prod__v3_realtime" -> HashingFeatureParams( scales = Seq(1487022661L, 1399245971L), biases = Seq(1372992088L, 632996194L), modulus = 2865175829L, bucketSize = 10000000L, ), "hr_video_prod__v2_lembeds" -> HashingFeatureParams( scales = Seq(2516541900L, 2376187492L), biases = Seq(3022238687L, 1571354734L), modulus = 3047336911L, bucketSize = 1000000L, ), "hr_prod__v4_embeds_230M" -> HashingFeatureParams( scales = Seq(2161410491L, 1754358832L), biases = Seq(296686044L, 1959990826L), modulus = 2361375383L, bucketSize = 100000000L, ), "hr_prod__v5_embeds_230M_and_transformer" -> HashingFeatureParams( scales = Seq(2161410491L, 1754358832L), biases = Seq(296686044L, 1959990826L), modulus = 2361375383L, bucketSize = 100000000L, ), "hr_prod__v5_watchtime" -> HashingFeatureParams( scales = Seq(407033648L, 940305868L), biases = Seq(494266171L, 269596788L), modulus = 949146421L, bucketSize = 100000000L, ), "hr_prod__v6_transformer_v2" -> HashingFeatureParams( scales = Seq(2161410491L, 1754358832L), biases = Seq(296686044L, 1959990826L), modulus = 2361375383L, bucketSize = 100000000L, ), "hr_prod__v6_mixed_training" -> HashingFeatureParams( scales = Seq(2161410491L, 1754358832L), biases = Seq(296686044L, 1959990826L), modulus = 2361375383L, bucketSize = 100000000L, ), "hr_prod__v6_transformer_v2_kafka_merge_join" -> HashingFeatureParams( scales = Seq(2161410491L, 1754358832L), biases = Seq(296686044L, 1959990826L), modulus = 2361375383L, bucketSize = 100000000L, ), "hr_prod__v6_transformer_v2_realtime_debias_21apr" -> HashingFeatureParams( scales = Seq(2161410491L, 1754358832L), biases = Seq(296686044L, 1959990826L), modulus = 2361375383L, bucketSize = 100000000L, ), "hr_video_prod__v4_realtime" -> HashingFeatureParams( scales = Seq(1487022661L, 1399245971L), biases = Seq(1372992088L, 632996194L), modulus = 2865175829L, bucketSize = 10000000L, ), "hr_video_prod__v4_realtime_mergehead" -> HashingFeatureParams( scales = Seq(1487022661L, 1399245971L), biases = Seq(1372992088L, 632996194L), modulus = 2865175829L, bucketSize = 10000000L, ), ) private val batchSize = 25 private def getBatchedFeatureMap( modelName: String, candidatesBatch: Seq[CandidateWithFeatures[TweetCandidate]], ): Future[Seq[FeatureMap]] = { val originalTweetIds = candidatesBatch.map { candidate => getOriginalTweetId(candidate.candidate, candidate.features) } getLargeEmbeddings(originalTweetIds, modelName).map { responses => responses.map { largeEmbeddingResponse => FeatureMapBuilder() .add(OriginalTweetLargeEmbeddingsFeature, largeEmbeddingResponse.dataRecord) .add(OriginalTweetLargeEmbeddingsKeyFeature, largeEmbeddingResponse.hashedKeys) .build() } } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val modelName = query.params(ModelNameParam) OffloadFuturePools.offloadBatchSeqToFutureSeq( candidates, getBatchedFeatureMap(modelName, _), batchSize ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.conversions.DurationOps._ import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.home_mixer.model.HomeFeatures.ServedAuthorIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.param.HomeGlobalParams.ExcludeServedAuthorIdsDurationParam import com.twitter.home_mixer.param.HomeGlobalParams.ExcludeServedTweetIdsDurationParam import com.twitter.home_mixer.param.HomeGlobalParams.ExcludeServedTweetIdsNumberParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelinemixer.clients.persistence.TimelineResponseBatchesClient import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 import com.twitter.timelines.util.client_info.ClientPlatform import com.twitter.timelineservice.model.TimelineQuery import com.twitter.timelineservice.model.core.TimelineKind import com.twitter.timelineservice.model.rich.EntityIdType import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton @Singleton case class PersistenceStoreQueryFeatureHydrator @Inject() ( timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3], statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore") private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val servedTweetIdsSizeStat = scopedStatsReceiver.stat("ServedTweetIdsSize") private val WhoToFollowExcludedUserIdsLimit = 1000 private val ServedTweetPreviewIdsDuration = 10.hours private val ServedTweetPreviewIdsLimit = 10 override val features: Set[Feature[_, _]] = Set( ServedTweetIdsFeature, ServedTweetPreviewIdsFeature, PersistenceEntriesFeature, WhoToFollowExcludedUserIdsFeature, ServedAuthorIdsFeature ) private val supportedClients = Seq( ClientPlatform.IPhone, ClientPlatform.IPad, ClientPlatform.Mac, ClientPlatform.Android, ClientPlatform.Web, ClientPlatform.RWeb, ClientPlatform.TweetDeckGryphon ) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val timelineKind = query.product match { case FollowingProduct => TimelineKind.homeLatest case ForYouProduct => TimelineKind.home case other => throw new UnsupportedOperationException(s"Unknown product: $other") } val timelineQuery = TimelineQuery(id = query.getRequiredUserId, kind = timelineKind) Stitch.callFuture { timelineResponseBatchesClient .get(query = timelineQuery, clientPlatforms = supportedClients) .map { timelineResponses => // Note that the WTF entries are not being scoped by ClientPlatform val whoToFollowUserIds = timelineResponses .flatMap { timelineResponse => timelineResponse.entries .filter(_.entityIdType == EntityIdType.WhoToFollow) .flatMap(_.itemIds.toSeq.flatMap(_.flatMap(_.userId))) }.take(WhoToFollowExcludedUserIdsLimit) val clientPlatform = ClientPlatform.fromQueryOptions( clientAppId = query.clientContext.appId, userAgent = query.clientContext.userAgent.flatMap(UserAgent.fromString)) val recentTimelineItems = timelineResponses .filter(_.clientPlatform == clientPlatform) .sortBy(-_.servedTime.inMilliseconds) val servedTweetIds = recentTimelineItems .filter(_.servedTime >= Time.now - query.params(ExcludeServedTweetIdsDurationParam)) .flatMap(_.entries .flatMap(_.tweetIds(includeSourceTweets = true)).take( query.params(ExcludeServedTweetIdsNumberParam))) servedTweetIdsSizeStat.add(servedTweetIds.size) val servedTweetPreviewIds = recentTimelineItems .filter(_.servedTime >= Time.now - ServedTweetPreviewIdsDuration) .flatMap( _.entries .filter(_.entityIdType == EntityIdType.TweetPreview) .flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetPreviewIdsLimit)) val servedAuthorIds: Map[Long, Seq[Long]] = recentTimelineItems .filter(_.servedTime >= Time.now - query.params(ExcludeServedAuthorIdsDurationParam)) .flatMap { timelineResponse => timelineResponse.entries .filter(_.entityIdType == EntityIdType.Tweet) .flatMap { entry => val authorId = entry.sourceAuthorIds.headOption.getOrElse(-1L) if (authorId != -1L) { // only include entries with valid author IDs entry.tweetIds(includeSourceTweets = true).map(tweetId => (authorId, tweetId)) } else Seq.empty } } .groupBy(_._1) .mapValues(_.map(_._2).distinct) FeatureMapBuilder() .add(ServedTweetIdsFeature, servedTweetIds) .add(ServedTweetPreviewIdsFeature, servedTweetPreviewIds) .add(PersistenceEntriesFeature, timelineResponses) .add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds) .add(ServedAuthorIdsFeature, servedAuthorIds) .build() } } } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.7, 50, 60, 60) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.stitch.timelineservice.TimelineService import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives import com.twitter.timelineservice.thriftscala.PerspectiveType import com.twitter.timelineservice.thriftscala.PerspectiveType.Favorited import javax.inject.Inject import javax.inject.Singleton /** * Filter out unlike edges from liked-by tweets * Useful if the likes come from a cache and because UTEG does not fully remove unlike edges. */ @Singleton class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService: TimelineService) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PerspectiveFilteredSocialContext") override val features: Set[Feature[_, _]] = Set(PerspectiveFilteredLikedByUserIdsFeature) private val MaxCountUsers = 10 private val favoritePerspectiveSet: Set[PerspectiveType] = Set(Favorited) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { val engagingUserIdtoTweetId = candidates.flatMap { candidate => candidate.features .getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers) .map(favoritedBy => favoritedBy -> candidate.candidate.id) } val queries = engagingUserIdtoTweetId.map { case (userId, tweetId) => GetPerspectives.Query(userId = userId, tweetId = tweetId, types = favoritePerspectiveSet) } Stitch.collect(queries.map(timelineService.getPerspective)).map { perspectiveResults => val validUserIdTweetIds: Set[(Long, Long)] = queries .zip(perspectiveResults) .collect { case (query, perspective) if perspective.favorited => query } .map(query => (query.userId, query.tweetId)) .toSet candidates.map { candidate => val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features .getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers) .filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) } FeatureMapBuilder() .add(PerspectiveFilteredLikedByUserIdsFeature, perspectiveFilteredFavoritedByUserIds) .build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PhoenixRescoringFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.PhoenixScoreFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnablePhoenixScorerParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnablePhoenixRescoreParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnablePhoenixScoreParam import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object PhoenixRescoringFeatureHydrator extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PhoenixRescoring") override val features: Set[Feature[_, _]] = Set(ScoreFeature) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { // Use rescoring to change scoreFeature only when scoring didn't happen using Phoenix val usePhoenixRescoring = query.params(EnablePhoenixScorerParam) && query.params(EnablePhoenixRescoreParam) && !query.params(EnablePhoenixScoreParam) val finalScores = candidates.map { candidate => val score = candidate.features.getOrElse(ScoreFeature, None).getOrElse(0.0) val weightedModelScore = candidate.features.getOrElse(WeightedModelScoreFeature, None).getOrElse(0.0) val phoenixScore = candidate.features.getOrElse(PhoenixScoreFeature, None).getOrElse(0.0) // The multiplier is for heuristics, it might not always be accurate for listwise heuristics if (score == 0.0 | weightedModelScore == 0.0) 0.0 else if (usePhoenixRescoring) phoenixScore * (score / weightedModelScore) else score } Stitch.value(finalScores.map(score => FeatureMap(ScoreFeature, Some(score)))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PostContextFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.GenericPostContextFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.param.HomeGlobalParams.MaxPostContextDuplicatesPerRequest import com.twitter.home_mixer.param.HomeGlobalParams.MaxPostContextPostsPerRequest import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.timelines.render.{thriftscala => urt} import com.twitter.strato.catalog.Fetch import com.twitter.strato.generated.client.events.urt.PostContextClientColumn import javax.inject.Inject import javax.inject.Singleton import scala.util.Random @Singleton class PostContextFeatureHydrator @Inject() ( postContextClientColumn: PostContextClientColumn) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PostContext") override val features: Set[Feature[_, _]] = Set(GenericPostContextFeature) private val fetcher: Fetcher[Long, Unit, urt.GenericContext] = postContextClientColumn.fetcher override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { val maxPosts = query.params(MaxPostContextPostsPerRequest) val maxDuplicates = query.params(MaxPostContextDuplicatesPerRequest) val contextsStitch: Stitch[Seq[Option[urt.GenericContext]]] = Stitch.collect { candidates.map { candidate => val servedType = candidate.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined) val isPromoted = { servedType == hmt.ServedType.ForYouPromoted || servedType == hmt.ServedType.FollowingPromoted } val isOriginal = (!candidate.features.getOrElse(IsRetweetFeature, false)) && candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty if (isPromoted || !isOriginal) { // Promoted tweets are not eligible for Post Context. Stitch.value(None) } else { fetcher.fetch(candidate.candidate.id, ()).map { case Fetch.Result(Some(postContext), _) => Some(postContext) case _ => None } } } } contextsStitch.map { rawContexts => val urlCount = scala.collection.mutable.Map.empty[String, Int] val afterDupFilter: Vector[Option[urt.GenericContext]] = rawContexts.map { case Some(ctx) => val url = ctx.url.url val seen = urlCount.getOrElse(url, 0) if (seen < maxDuplicates) { urlCount.update(url, seen + 1) Some(ctx) } else None // drop: duplicate overflow case None => None }(collection.breakOut) val keptIndices: Vector[Int] = afterDupFilter.iterator.zipWithIndex.collect { case (Some(_), idx) => idx }.toVector val indicesToDrop: Set[Int] = if (keptIndices.size <= maxPosts) Set.empty else { val numToDrop = keptIndices.size - maxPosts Random.shuffle(keptIndices).take(numToDrop).toSet } afterDupFilter.zipWithIndex.map { case (Some(ctx), idx) if !indicesToDrop.contains(idx) => FeatureMapBuilder().add(GenericPostContextFeature, Some(ctx)).build() case _ => FeatureMapBuilder().add(GenericPostContextFeature, None).build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RateLimitQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.ViewerIsRateLimited import com.twitter.home_mixer.param.HomeGlobalParams.RateLimitTestIdsParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.limiter.{thriftscala => t} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton case class RateLimitQueryFeatureHydrator @Inject() (limiterClient: t.LimitService.MethodPerEndpoint) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RateLimit") override val features: Set[Feature[_, _]] = Set(ViewerIsRateLimited) val RegularFeature = "graphql_global_regular_tweets_read" val NewFeature = "graphql_global_new_tweets_read" val SoftFeature = "graphql_global_soft_user_tweets_read" val SuspendedFeature = "graphql_global_suspended_user_tweets_read" override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getOptionalUserId val appId = query.clientContext.appId val rateLimitTestIds = query.params(RateLimitTestIdsParam) val usagesStitch = Seq( t.FeatureRequest(RegularFeature, userId, applicationId = appId), t.FeatureRequest(NewFeature, userId, applicationId = appId), t.FeatureRequest(SoftFeature, userId, applicationId = appId), t.FeatureRequest(SuspendedFeature, userId, applicationId = appId) ).map { request => Stitch.callFuture(limiterClient.getLimitUsage(None, Some(request))) } Stitch.collect(usagesStitch).map { usage => val limited = if (rateLimitTestIds.contains(userId.get)) usage.map(u => u.remaining.toDouble / u.limit).exists(_ < 0.999) else usage.map(_.remaining).exists(_ == 0) FeatureMapBuilder().add(ViewerIsRateLimited, limited).build() } } override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(70)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphInNetworkScoresOnPrem import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.storehaus.ReadableStore import com.twitter.wtf.candidate.{thriftscala => wtf} import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() ( @Named(RealGraphInNetworkScoresOnPrem) realGraphMhStore: ReadableStore[Long, Seq[wtf.Candidate]]) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RealGraphInNetworkScores") override val features: Set[Feature[_, _]] = Set(RealGraphInNetworkScoresFeature) private val RealGraphCandidateCount = 1000 override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { Stitch.callFuture(realGraphMhStore.get(query.getRequiredUserId)).map { realGraphFollowedUsers => val realGraphScoresFeatures = realGraphFollowedUsers .getOrElse(Seq.empty) .sortBy(-_.score) .map(candidate => candidate.userId -> scaleScore(candidate.score)) .take(RealGraphCandidateCount) .toMap FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build() } } // Rescale Real Graph v2 scores from [0,1] to the v1 scores distribution [1,2.97] // v1 logic: src/scala/com/twitter/interaction_graph/scalding/jobs/scoring/InteractionGraphScoringJob.scala?L77-80 private def scaleScore(score: Double): Double = if (score >= 0.0 && score <= 1.0) score * 1.97 + 1.0 else score } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableOnPremRealGraphQueryFeatures import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphFeatureRepository import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.repository.Repository import com.twitter.stitch.Stitch import com.twitter.timelines.model.UserId import com.twitter.timelines.real_graph.v1.thriftscala.RealGraphEdgeFeatures import com.twitter.timelines.real_graph.{thriftscala => rg} import com.twitter.user_session_store.{thriftscala => uss} import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton object RealGraphFeatures extends Feature[PipelineQuery, Option[Map[UserId, RealGraphEdgeFeatures]]] @Singleton class RealGraphQueryFeatureHydrator @Inject() ( @Named(RealGraphFeatureRepository) repository: Repository[Long, Option[uss.UserSession]]) extends QueryFeatureHydrator[PipelineQuery] with Conditionally[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RealGraphFeatures") override val features: Set[Feature[_, _]] = Set(RealGraphFeatures) override def onlyIf( query: PipelineQuery ): Boolean = { !query.params(EnableOnPremRealGraphQueryFeatures) } override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { Stitch.callFuture { repository(query.getRequiredUserId).map { userSession => val realGraphFeaturesMap = userSession.flatMap { userSession => userSession.realGraphFeatures.collect { case rg.RealGraphFeatures.V1(realGraphFeatures) => val edgeFeatures = realGraphFeatures.edgeFeatures ++ realGraphFeatures.oonEdgeFeatures edgeFeatures.map { edge => edge.destId -> edge }.toMap } } FeatureMapBuilder().add(RealGraphFeatures, realGraphFeaturesMap).build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.functional_component.feature_hydrator.RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature import com.twitter.home_mixer.util.MissingKeyException import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter import com.twitter.timelines.prediction.adapters.real_graph.RealGraphFeaturesAdapter import com.twitter.timelines.real_graph.v1.{thriftscala => v1} import com.twitter.timelines.real_graph.{thriftscala => rg} import com.twitter.util.Throw import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ object RealGraphViewerAuthorDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object RealGraphViewerAuthorsDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class RealGraphViewerAuthorFeatureHydrator @Inject() () extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RealGraphViewerAuthor") override val features: Set[Feature[_, _]] = Set(RealGraphViewerAuthorDataRecordFeature, RealGraphViewerAuthorsDataRecordFeature) private val realGraphEdgeFeaturesAdapter = new RealGraphFeaturesAdapter private val realGraphEdgeFeaturesCombineAdapter = new RealGraphEdgeFeaturesCombineAdapter(prefix = "authors.realgraph") private val MissingKeyFeatureMap = FeatureMapBuilder() .add(RealGraphViewerAuthorDataRecordFeature, Throw(MissingKeyException)) .add(RealGraphViewerAuthorsDataRecordFeature, Throw(MissingKeyException)) .build() private val batchSize = 64 def getFeatureMap( candidate: CandidateWithFeatures[TweetCandidate], viewerId: Long, realGraphFeatures: Map[Long, v1.RealGraphEdgeFeatures] ): FeatureMap = candidate.features.getOrElse(AuthorIdFeature, None) match { case Some(authorId) => val realGraphAuthorFeatures = getRealGraphViewerAuthorFeatures(viewerId, authorId, realGraphFeatures) val realGraphAuthorDataRecord = realGraphEdgeFeaturesAdapter .adaptToDataRecords(realGraphAuthorFeatures).asScala.headOption.getOrElse(new DataRecord) val combinedRealGraphFeaturesDataRecord = for { inReplyToAuthorId <- candidate.features.getOrElse(InReplyToUserIdFeature, None) } yield { val combinedRealGraphFeatures = getCombinedRealGraphFeatures(Seq(authorId, inReplyToAuthorId), realGraphFeatures) realGraphEdgeFeaturesCombineAdapter .adaptToDataRecords(Some(combinedRealGraphFeatures)).asScala.headOption .getOrElse(new DataRecord) } FeatureMap( RealGraphViewerAuthorDataRecordFeature, realGraphAuthorDataRecord, RealGraphViewerAuthorsDataRecordFeature, combinedRealGraphFeaturesDataRecord.getOrElse(new DataRecord) ) case _ => MissingKeyFeatureMap } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val viewerId = query.getRequiredUserId val realGraphFeatures = query.features .flatMap(_.getOrElse(RealGraphFeatures, None)) .getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures]) OffloadFuturePools.offloadBatchElementToElement( candidates, getFeatureMap(_, viewerId, realGraphFeatures), batchSize) } private def getRealGraphViewerAuthorFeatures( viewerId: Long, authorId: Long, realGraphEdgeFeaturesMap: Map[Long, v1.RealGraphEdgeFeatures] ): rg.UserRealGraphFeatures = { realGraphEdgeFeaturesMap.get(authorId) match { case Some(realGraphEdgeFeatures) => rg.UserRealGraphFeatures( srcId = viewerId, features = rg.RealGraphFeatures.V1( v1.RealGraphFeatures(edgeFeatures = Seq(realGraphEdgeFeatures)))) case _ => rg.UserRealGraphFeatures( srcId = viewerId, features = rg.RealGraphFeatures.V1(v1.RealGraphFeatures(edgeFeatures = Seq.empty))) } } } object RealGraphViewerAuthorFeatureHydrator { def getCombinedRealGraphFeatures( userIds: Seq[Long], realGraphEdgeFeaturesMap: Map[Long, v1.RealGraphEdgeFeatures] ): rg.RealGraphFeatures = { val edgeFeatures = userIds.flatMap(realGraphEdgeFeaturesMap.get) rg.RealGraphFeatures.V1(v1.RealGraphFeatures(edgeFeatures = edgeFeatures)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableRealGraphViewerRelatedUsersFeaturesParam import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter import com.twitter.timelines.real_graph.v1.{thriftscala => v1} import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ object RealGraphViewerRelatedUsersDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class RealGraphViewerRelatedUsersFeatureHydrator @Inject() () extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] with WithDefaultFeatureMap { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RealGraphViewerRelatedUsers") override val features: Set[Feature[_, _]] = Set(RealGraphViewerRelatedUsersDataRecordFeature) override val defaultFeatureMap: FeatureMap = FeatureMap( RealGraphViewerRelatedUsersDataRecordFeature, RealGraphViewerRelatedUsersDataRecordFeature.defaultValue) private val RealGraphEdgeFeaturesCombineAdapter = new RealGraphEdgeFeaturesCombineAdapter override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableRealGraphViewerRelatedUsersFeaturesParam) val batchSize = 64 def getFeatureMap( candidate: CandidateWithFeatures[TweetCandidate], realGraphQueryFeatures: Map[Long, v1.RealGraphEdgeFeatures] ): FeatureMap = { val allRelatedUserIds = getRelatedUserIds(candidate.features) val realGraphFeatures = RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures( allRelatedUserIds, realGraphQueryFeatures ) val realGraphFeaturesDataRecord = RealGraphEdgeFeaturesCombineAdapter .adaptToDataRecords(Some(realGraphFeatures)).asScala.headOption .getOrElse(new DataRecord) FeatureMap(RealGraphViewerRelatedUsersDataRecordFeature, realGraphFeaturesDataRecord) } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val realGraphQueryFeatures = query.features .flatMap(_.getOrElse(RealGraphFeatures, None)) .getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures]) OffloadFuturePools.offloadBatchElementToElement( candidates, getFeatureMap(_, realGraphQueryFeatures), batchSize) } private def getRelatedUserIds(features: FeatureMap): Seq[Long] = { (CandidatesUtil.getEngagerUserIds(features) ++ features.getOrElse(AuthorIdFeature, None) ++ features.getOrElse(MentionUserIdFeature, Seq.empty) ++ features.getOrElse(SourceUserIdFeature, None) ++ features.getOrElse(DirectedAtUserIdFeature, None)).distinct } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeEntityRealGraphQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.EntityRealGraphClientStore import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.recos.entities.{thriftscala => ent} import com.twitter.stitch.Stitch import com.twitter.storehaus.ReadableStore import com.twitter.wtf.entity_real_graph.{thriftscala => erg} import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton object RealTimeEntityRealGraphFeatures extends Feature[PipelineQuery, Option[Map[ent.Entity, Map[erg.EngagementType, erg.Features]]]] @Singleton class RealTimeEntityRealGraphQueryFeatureHydrator @Inject() ( @Named(EntityRealGraphClientStore) client: ReadableStore[ erg.EntityRealGraphRequest, erg.EntityRealGraphResponse ], override val statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RealTimeEntityRealGraphFeatures") override val features: Set[Feature[_, _]] = Set(RealTimeEntityRealGraphFeatures) override val statScope: String = identifier.toString override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val entityRealGraphRequest = erg.EntityRealGraphRequest( userId = query.getRequiredUserId, entityTypes = Set(erg.EntityType.SemanticCore), normalizeCounts = Some(true)) Stitch.callFuture { client.get(entityRealGraphRequest).map { response => val engagements = response.map(_.response.mapValues(_.toMap).toMap) FeatureMapBuilder().add(RealTimeEntityRealGraphFeatures, engagements).build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.realtime_interaction_graph.RealTimeInteractionGraphFeaturesAdapter import com.twitter.timelines.prediction.features.realtime_interaction_graph.RealTimeInteractionGraphEdgeFeatures import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ object RealTimeInteractionGraphEdgeFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class RealTimeInteractionGraphEdgeFeatureHydrator @Inject() () extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RealTimeInteractionGraphEdge") override val features: Set[Feature[_, _]] = Set(RealTimeInteractionGraphEdgeFeature) private val realTimeInteractionGraphFeaturesAdapter = new RealTimeInteractionGraphFeaturesAdapter override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { val userVertex = query.features.flatMap(_.getOrElse(RealTimeInteractionGraphUserVertexQueryFeature, None)) val realTimeInteractionGraphFeaturesMap = userVertex.map(RealTimeInteractionGraphEdgeFeatures(_, Time.now)) candidates.map { candidate => val feature = candidate.features.getOrElse(AuthorIdFeature, None).flatMap { authorId => realTimeInteractionGraphFeaturesMap.flatMap(_.get(authorId)) } val dataRecordFeature = realTimeInteractionGraphFeaturesAdapter.adaptToDataRecords(feature).asScala.head FeatureMap(RealTimeInteractionGraphEdgeFeature, dataRecordFeature) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionGraphUserVertexCache import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.ReadCache import com.twitter.stitch.Stitch import com.twitter.wtf.real_time_interaction_graph.{thriftscala => ig} import javax.inject.Inject import javax.inject.Singleton object RealTimeInteractionGraphUserVertexQueryFeature extends Feature[PipelineQuery, Option[ig.UserVertex]] @Singleton class RealTimeInteractionGraphUserVertexQueryFeatureHydrator @Inject() ( @Named(RealTimeInteractionGraphUserVertexCache) client: ReadCache[Long, ig.UserVertex], override val statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RealTimeInteractionGraphUserVertex") override val features: Set[Feature[_, _]] = Set(RealTimeInteractionGraphUserVertexQueryFeature) override val statScope: String = identifier.toString override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId Stitch.callFuture( client.get(Seq(userId)).map { results => val feature = observedGet(key = Some(userId), keyValueResult = results) FeatureMapBuilder() .add(RealTimeInteractionGraphUserVertexQueryFeature, feature) .build() } ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.tracing.Annotation.BinaryAnnotation import com.twitter.finagle.tracing.ForwardAnnotation import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.model.request.DeviceContext.RequestContext import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.joinkey.context.RequestJoinKeyContext import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor import com.twitter.product_mixer.core.pipeline.HasPipelineCursor import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.search.common.util.lang.ThriftLanguageUtil import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.dowFromTimestamp import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.hourFromTimestamp import java.util.UUID import javax.inject.Inject import javax.inject.Singleton @Singleton class RequestQueryFeatureHydrator[ Query <: PipelineQuery with HasPipelineCursor[UrtOrderedCursor] with HasDeviceContext] @Inject() ( ) extends QueryFeatureHydrator[Query] { override val features: Set[Feature[_, _]] = Set( AccountAgeFeature, ClientIdFeature, DeviceCountryFeature, DeviceLanguageFeature, GetInitialFeature, GetMiddleFeature, GetNewerFeature, GetOlderFeature, GuestIdFeature, HasDarkRequestFeature, IsForegroundRequestFeature, IsLaunchRequestFeature, PollingFeature, PullToRefreshFeature, RequestJoinIdFeature, ServedIdFeature, TimestampFeature, TimestampGMTDowFeature, TimestampGMTHourFeature, ViewerIdFeature ) override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Request") private val DarkRequestAnnotation = "clnt/has_dark_request" // Convert Language code to ISO 639-3 format private def getLanguageISOFormatByCode(languageCode: String): String = ThriftLanguageUtil.getLanguageCodeOf(ThriftLanguageUtil.getThriftLanguageOf(languageCode)) private def getRequestJoinId: Option[Long] = RequestJoinKeyContext.current.flatMap(_.requestJoinId) private def hasDarkRequest: Option[Boolean] = ForwardAnnotation.current .getOrElse(Seq[BinaryAnnotation]()) .find(_.key == DarkRequestAnnotation) .map(_.value.asInstanceOf[Boolean]) override def hydrate(query: Query): Stitch[FeatureMap] = { val requestContext = query.deviceContext.flatMap(_.requestContextValue) val servedId = UUID.randomUUID.getMostSignificantBits val timestamp = query.queryTime.inMilliseconds val featureMap = FeatureMapBuilder() .add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt)) .add(ClientIdFeature, query.clientContext.appId) .add(DeviceCountryFeature, query.getCountryCode) .add(DeviceLanguageFeature, query.getLanguageCode.map(getLanguageISOFormatByCode)) .add( GetInitialFeature, query.pipelineCursor.forall(cursor => cursor.id.isEmpty && cursor.gapBoundaryId.isEmpty)) .add( GetMiddleFeature, query.pipelineCursor.exists(cursor => cursor.id.isDefined && cursor.gapBoundaryId.isDefined && cursor.cursorType.contains(GapCursor))) .add( GetNewerFeature, query.pipelineCursor.exists(cursor => cursor.id.isDefined && cursor.gapBoundaryId.isEmpty && cursor.cursorType.contains(TopCursor))) .add( GetOlderFeature, query.pipelineCursor.exists(cursor => cursor.id.isDefined && cursor.gapBoundaryId.isEmpty && cursor.cursorType.contains(BottomCursor))) .add(GuestIdFeature, query.clientContext.guestId) .add(IsForegroundRequestFeature, requestContext.contains(RequestContext.Foreground)) .add(IsLaunchRequestFeature, requestContext.contains(RequestContext.Launch)) .add(PollingFeature, query.deviceContext.exists(_.isPolling.contains(true))) .add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh)) .add(ServedIdFeature, Some(servedId)) .add(RequestJoinIdFeature, getRequestJoinId) .add(TimestampFeature, timestamp) .add(TimestampGMTDowFeature, dowFromTimestamp(timestamp)) .add(TimestampGMTHourFeature, hourFromTimestamp(timestamp)) .add(HasDarkRequestFeature, hasDarkRequest) .add( ViewerIdFeature, query.getOptionalUserId .orElse(query.getGuestId).getOrElse( throw PipelineFailure(BadRequest, "Missing viewer id"))) .build() Stitch.value(featureMap) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestTimeQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.FollowingLastNonPollingTimeFeature import com.twitter.home_mixer.model.HomeFeatures.LastNonPollingTimeFeature import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature import com.twitter.ml.api.DataRecord import com.twitter.ml.api.util.FDsl._ import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.ACCOUNT_AGE_INTERVAL import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.IS_12_MONTH_NEW_USER import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.IS_30_DAY_NEW_USER import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_BETWEEN_NON_POLLING_REQUESTS_AVG import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_SINCE_LAST_NON_POLLING_REQUEST import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.USER_ID_IS_SNOWFLAKE_ID import com.twitter.user_session_store.ReadRequest import com.twitter.user_session_store.ReadWriteUserSessionStore import com.twitter.user_session_store.UserSessionDataset import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton object RequestTimeDataRecordFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton case class RequestTimeQueryFeatureHydrator @Inject() ( userSessionStore: ReadWriteUserSessionStore) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RequestTime") override val features: Set[Feature[_, _]] = Set( FollowingLastNonPollingTimeFeature, LastNonPollingTimeFeature, NonPollingTimesFeature, RequestTimeDataRecordFeature ) private val datasets: Set[UserSessionDataset] = Set(UserSessionDataset.NonPollingTimes) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { userSessionStore .read(ReadRequest(query.getRequiredUserId, datasets)) .map { userSession => val nonPollingTimestamps = userSession.flatMap(_.nonPollingTimestamps) val lastNonPollingTime = nonPollingTimestamps .flatMap(_.nonPollingTimestampsMs.headOption) .map(Time.fromMilliseconds) val followingLastNonPollingTime = nonPollingTimestamps .flatMap(_.mostRecentHomeLatestNonPollingTimestampMs) .map(Time.fromMilliseconds) val nonPollingTimes = nonPollingTimestamps .map(_.nonPollingTimestampsMs) .getOrElse(Seq.empty) val requestTimeDataRecord = getRequestTimeDataRecord(query, nonPollingTimes) FeatureMapBuilder() .add(FollowingLastNonPollingTimeFeature, followingLastNonPollingTime) .add(LastNonPollingTimeFeature, lastNonPollingTime) .add(NonPollingTimesFeature, nonPollingTimes) .add(RequestTimeDataRecordFeature, requestTimeDataRecord) .build() } } def getRequestTimeDataRecord(query: PipelineQuery, nonPollingTimes: Seq[Long]): DataRecord = { val requestTimeMs = query.queryTime.inMillis val accountAge = SnowflakeId.timeFromIdOpt(query.getRequiredUserId) val timeSinceAccountCreation = accountAge.map(query.queryTime.since) val timeSinceEarliestNonPollingRequest = nonPollingTimes.lastOption.map(requestTimeMs - _) val timeSinceLastNonPollingRequest = nonPollingTimes.headOption.map(requestTimeMs - _) new DataRecord() .setFeatureValue(USER_ID_IS_SNOWFLAKE_ID, accountAge.isDefined) .setFeatureValue( IS_30_DAY_NEW_USER, timeSinceAccountCreation.map(_ < 30.days).getOrElse(false) ) .setFeatureValue( IS_12_MONTH_NEW_USER, timeSinceAccountCreation.map(_ < 365.days).getOrElse(false) ) .setFeatureValueFromOption( ACCOUNT_AGE_INTERVAL, timeSinceAccountCreation.flatMap(AccountAgeInterval.fromDuration).map(_.id.toLong) ) .setFeatureValueFromOption( TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS, timeSinceAccountCreation.map(_.inSeconds.toDouble) ) .setFeatureValueFromOption( TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, timeSinceEarliestNonPollingRequest.map(_.toDouble / math.max(1.0, nonPollingTimes.size)) ) .setFeatureValueFromOption( TIME_SINCE_LAST_NON_POLLING_REQUEST, timeSinceLastNonPollingRequest.map(_.toDouble) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.socialgraph.{thriftscala => sg} import com.twitter.stitch.Stitch import com.twitter.stitch.socialgraph.SocialGraph import javax.inject.Inject import javax.inject.Singleton /** * This hydrator takes liked-by and followed-by user ids and checks via SGS that the viewer is * following the engager, that the viewer is not blocking the engager, that the engager is not * blocking the viewer, and that the viewer has not muted the engager. */ @Singleton class SGSValidSocialContextFeatureHydrator @Inject() ( socialGraph: SocialGraph) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with WithDefaultFeatureMap { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("SGSValidSocialContext") override val features: Set[Feature[_, _]] = Set( SGSValidFollowedByUserIdsFeature, SGSValidLikedByUserIdsFeature ) override val defaultFeatureMap: FeatureMap = FeatureMap( SGSValidFollowedByUserIdsFeature, Seq.empty, SGSValidLikedByUserIdsFeature, Seq.empty ) private val MaxCountUsers = 10 override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { val allSocialContextUserIds = candidates.flatMap { candidate => candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++ candidate.features.getOrElse(FollowedByUserIdsFeature, Nil).take(MaxCountUsers) }.distinct getValidUserIds(query.getRequiredUserId, allSocialContextUserIds).map { validUserIds => candidates.map { candidate => val sgsFilteredLikedByUserIds = candidate.features .getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) .filter(validUserIds.contains) val sgsFilteredFollowedByUserIds = candidate.features .getOrElse(FollowedByUserIdsFeature, Nil).take(MaxCountUsers) .filter(validUserIds.contains) FeatureMapBuilder() .add(SGSValidFollowedByUserIdsFeature, sgsFilteredFollowedByUserIds) .add(SGSValidLikedByUserIdsFeature, sgsFilteredLikedByUserIds) .build() } } } private def getValidUserIds( viewerId: Long, socialProofUserIds: Seq[Long] ): Stitch[Seq[Long]] = { if (socialProofUserIds.nonEmpty) { val request = sg.IdsRequest( relationships = Seq( sg.SrcRelationship( viewerId, sg.RelationshipType.Following, targets = Some(socialProofUserIds), hasRelationship = true), sg.SrcRelationship( viewerId, sg.RelationshipType.Blocking, targets = Some(socialProofUserIds), hasRelationship = false), sg.SrcRelationship( viewerId, sg.RelationshipType.BlockedBy, targets = Some(socialProofUserIds), hasRelationship = false), sg.SrcRelationship( viewerId, sg.RelationshipType.Muting, targets = Some(socialProofUserIds), hasRelationship = false) ), pageRequest = Some(sg.PageRequest(selectAll = Some(true))) ) socialGraph.ids(request).map(_.ids) } else Stitch.Nil } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableSimClustersSimilarityFeaturesDeciderParam import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.clients.strato.twistly.SimClustersRecentEngagementSimilarityClient import com.twitter.timelines.configapi.decider.BooleanDeciderParam import com.twitter.timelines.prediction.adapters.twistly.SimClustersRecentEngagementSimilarityFeaturesAdapter import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton object SimClustersEngagementSimilarityFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class SimClustersEngagementSimilarityFeatureHydrator @Inject() ( simClustersEngagementSimilarityClient: SimClustersRecentEngagementSimilarityClient) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("SimClustersEngagementSimilarity") override val features: Set[Feature[_, _]] = Set(SimClustersEngagementSimilarityFeature) private val simClustersRecentEngagementSimilarityFeaturesAdapter = new SimClustersRecentEngagementSimilarityFeaturesAdapter override def onlyIf(query: PipelineQuery): Boolean = { val param: BooleanDeciderParam = EnableSimClustersSimilarityFeaturesDeciderParam query.params.apply(param) } def getFeatureMaps( candidates: Seq[CandidateWithFeatures[TweetCandidate]], userId: Long ): Future[Seq[FeatureMap]] = { val tweetToCandidates = candidates.map(candidate => candidate.candidate.id -> candidate).toMap val tweetIds = tweetToCandidates.keySet.toSeq val userTweetEdges = tweetIds.map(tweetId => (userId, tweetId)) simClustersEngagementSimilarityClient .getSimClustersRecentEngagementSimilarityScores(userTweetEdges).map { simClustersRecentEngagementSimilarityScoresMap => candidates.map { candidate => val similarityFeatureOpt = simClustersRecentEngagementSimilarityScoresMap .get(userId -> candidate.candidate.id).flatten val dataRecordOpt = similarityFeatureOpt.map { similarityFeature => simClustersRecentEngagementSimilarityFeaturesAdapter .adaptToDataRecords(similarityFeature) .get(0) } FeatureMap( SimClustersEngagementSimilarityFeature, dataRecordOpt.getOrElse(new DataRecord)) } } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = Stitch.callFuture { getFeatureMaps(candidates, query.getRequiredUserId) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersLogFavBasedTweetFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.DiversityRescoringFeatureHydrator.EmptyDataRecord import com.twitter.home_mixer_features.thriftjava.HomeMixerFeaturesRequest import com.twitter.home_mixer_features.{thriftjava => t} import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ object SimClustersLogFavBasedTweetFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class SimClustersLogFavBasedTweetFeatureHydrator @Inject() ( homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient, statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "SimClustersLogFavBasedTweet") override val features: Set[Feature[_, _]] = Set(SimClustersLogFavBasedTweetFeature) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyTotalCounter = scopedStatsReceiver.counter("key/total") private val batchSize = 50 private def getEmbeddingsFromHMF( tweetIds: Seq[Long] ): Future[Seq[DataRecord]] = { val keysSerialized = tweetIds.map(_.toString) val request = new HomeMixerFeaturesRequest() request.setKeys(keysSerialized.asJava) request.setCache(t.Cache.LOG_FAV_BASED_TWEET_20M145K2020_EMBEDDINGS) homeMixerFeatureService .getHomeMixerFeatures(request) .map { resp => unmarshallHomeMixerFeaturesResponse(resp) } } private def unmarshallHomeMixerFeaturesResponse( response: t.HomeMixerFeaturesResponse ): Seq[DataRecord] = { response.getHomeMixerFeatures.asScala.map { homeMixerFeatureOpt => if (homeMixerFeatureOpt.isSetHomeMixerFeaturesType) { val homeMixerFeature = homeMixerFeatureOpt.getHomeMixerFeaturesType if (homeMixerFeature.isSet(t.HomeMixerFeaturesType._Fields.DATA_RECORD)) { homeMixerFeature.getDataRecord } else { throw new Exception("Unexpected type") } } else EmptyDataRecord } } private def getBatchedFeatureMap( candidatesBatch: Seq[CandidateWithFeatures[TweetCandidate]] ): Future[Seq[FeatureMap]] = { val tweetIds = candidatesBatch.map { candidate => keyTotalCounter.incr() candidate.candidate.id } getEmbeddingsFromHMF(tweetIds).map { response => response.map { dataRecordOpt => keyFoundCounter.incr() FeatureMap(SimClustersLogFavBasedTweetFeature, dataRecordOpt) } } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { OffloadFuturePools.offloadBatchSeqToFutureSeq(candidates, getBatchedFeatureMap, batchSize) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersUserSparseEmbeddingsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.dal.personal_data.{thriftjava => pd} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature import com.twitter.product_mixer.core.feature.datarecord.SparseContinuousDataRecordCompatible import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.recommendations.simclusters_v2.InterestedIn20M145K2020OnUserClientColumn import com.twitter.timelines.prediction.features.simcluster.SimclusterFeatures import javax.inject.Inject import javax.inject.Singleton object SimClustersUserLogFavSparseEmbeddingsDataRecordFeature extends DataRecordOptionalFeature[PipelineQuery, Map[String, Double]] with SparseContinuousDataRecordCompatible { override val featureName: String = SimclusterFeatures.SIMCLUSTER_USER_LOG_FAV_CLUSTER_SCORES.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.InferredInterests) } object SimClustersUserFollowSparseEmbeddingsDataRecordFeature extends DataRecordOptionalFeature[PipelineQuery, Map[String, Double]] with SparseContinuousDataRecordCompatible { override val featureName: String = SimclusterFeatures.SIMCLUSTER_USER_FOLLOW_CLUSTER_SCORES.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.InferredInterests) } @Singleton class SimClustersUserSparseEmbeddingsQueryFeatureHydrator @Inject() ( interestedIn20M145K2020OnUserClientColumn: InterestedIn20M145K2020OnUserClientColumn) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("SimClustersUserSparseEmbeddingsQuery") override val features: Set[Feature[_, _]] = Set( SimClustersUserLogFavSparseEmbeddingsDataRecordFeature, SimClustersUserFollowSparseEmbeddingsDataRecordFeature) private val DefaultFeatureMap = FeatureMapBuilder() .add(SimClustersUserLogFavSparseEmbeddingsDataRecordFeature, None) .add(SimClustersUserFollowSparseEmbeddingsDataRecordFeature, None) .build() override def hydrate( query: PipelineQuery, ): Stitch[FeatureMap] = { val interestedInEmbeddingsOptStitch = interestedIn20M145K2020OnUserClientColumn.fetcher .fetch(query.getRequiredUserId) .map { result => result.v.map { interestedInEmbeddings => interestedInEmbeddings.clusterIdToScores } } interestedInEmbeddingsOptStitch .map { interestedInEmbeddingsOpt => val logFavEmbeddings = interestedInEmbeddingsOpt.map { interestedInEmbeddings => interestedInEmbeddings.map { case (key, value) => (key.toString, value.logFavScore.getOrElse(0.0)) }.toMap } val followEmbeddings = interestedInEmbeddingsOpt.map { interestedInEmbeddings => interestedInEmbeddings.map { case (key, value) => (key.toString, value.followScore.getOrElse(0.0)) }.toMap } FeatureMapBuilder() .add(SimClustersUserLogFavSparseEmbeddingsDataRecordFeature, logFavEmbeddings) .add(SimClustersUserFollowSparseEmbeddingsDataRecordFeature, followEmbeddings) .build() }.handle { case _ => DefaultFeatureMap } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimClustersUserTweetScoresHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.dal.personal_data.{thriftjava => pd} import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.strato.catalog.Fetch import com.twitter.strato.generated.client.ml.featureStore.SimClustersUserInterestedInTweetEmbeddingDotProduct20M145K2020OnUserTweetEdgeClientColumn import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton object SimClustersUserInterestedInTweetEmbeddingDataRecordFeature extends DataRecordOptionalFeature[TweetCandidate, Double] with DoubleDataRecordCompatible { override val featureName: String = "user-tweet.recommendations.sim_clusters_scores.user_interested_in_tweet_embedding_dot_product_20m_145k_2020" override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.InferredInterests) } @Singleton class SimClustersUserTweetScoresHydrator @Inject() ( simClustersColumn: SimClustersUserInterestedInTweetEmbeddingDotProduct20M145K2020OnUserTweetEdgeClientColumn, statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("SimClustersUserTweetScores") override val features: Set[Feature[_, _]] = Set(SimClustersUserInterestedInTweetEmbeddingDataRecordFeature) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") private val keySkipCounter = scopedStatsReceiver.counter("key/skip") private val DefaultFeatureMap = FeatureMapBuilder() .add(SimClustersUserInterestedInTweetEmbeddingDataRecordFeature, None) .build() private val MinFavToHydrate = 9 private val batchSize = 64 def getFeatureMaps( candidates: Seq[CandidateWithFeatures[TweetCandidate]], userId: Long ): Future[Seq[FeatureMap]] = { val featureMapStitch = Stitch.traverse(candidates) { candidate => val ebFeatures = candidate.features.getOrElse(EarlybirdFeature, None) val favCount = ebFeatures.flatMap(_.favCountV2).getOrElse(0) if (ebFeatures.isEmpty || favCount >= MinFavToHydrate) { simClustersColumn.fetcher .fetch((userId, candidate.candidate.id), Unit) .map { case Fetch.Result(response, _) => if (response.nonEmpty) keyFoundCounter.incr() else keyNotFoundCounter.incr() FeatureMapBuilder() .add(SimClustersUserInterestedInTweetEmbeddingDataRecordFeature, response) .build() case _ => keyFailureCounter.incr() DefaultFeatureMap } } else { keySkipCounter.incr() Stitch.value(DefaultFeatureMap) } } Stitch.run(featureMapStitch) } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { OffloadFuturePools.offloadBatchSeqToFutureSeq( candidates, getFeatureMaps(_, query.getRequiredUserId), batchSize, offload = true) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SimclusterBasedTopAuthorsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.ExpiringLruInProcessCache import com.twitter.servo.cache.InProcessCache import com.twitter.simclusters_v2.thriftscala.ClusterDetails import com.twitter.stitch.Stitch import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton import scala.util.Random object SimclustersFavBasedTopAuthors extends Feature[PipelineQuery, Seq[(Long, Double)]] object SimclustersFollowBasedTopAuthors extends Feature[PipelineQuery, Seq[(Long, Double)]] object SimclusterBasedTopAuthorsQueryFeatureHydrator { private val BaseTTL = 60 * 24 private val TTL = (BaseTTL + Random.nextInt(60)).minutes val cache: InProcessCache[String, Seq[(Long, Double)]] = new ExpiringLruInProcessCache(ttl = TTL, maximumSize = 150 * 1000) } @Singleton class SimclusterBasedTopAuthorsQueryFeatureHydrator @Inject() ( store: ReadableStore[String, ClusterDetails]) extends QueryFeatureHydrator[PipelineQuery] { import SimclusterBasedTopAuthorsQueryFeatureHydrator._ override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("SimclusterBasedTopAuthors") override val features: Set[Feature[_, _]] = Set(SimclustersFavBasedTopAuthors, SimclustersFollowBasedTopAuthors) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val favBasedEmbeddings = query.features .flatMap(_.getOrElse(SimClustersUserLogFavSparseEmbeddingsDataRecordFeature, None)) .getOrElse(Map.empty[String, Double]) val followBasedEmbeddings = query.features .flatMap(_.getOrElse(SimClustersUserFollowSparseEmbeddingsDataRecordFeature, None)) .getOrElse(Map.empty[String, Double]) val favBasedTopAuthorsStitch = Stitch.callFuture(getTopAuthorsWithScores(favBasedEmbeddings)) val followBasedTopAuthorsStitch = Stitch.callFuture(getTopAuthorsWithScores(followBasedEmbeddings)) Stitch.join(favBasedTopAuthorsStitch, followBasedTopAuthorsStitch).map { case (favBasedTopAuthors, followBasedTopAuthors) => FeatureMap( SimclustersFavBasedTopAuthors, favBasedTopAuthors, SimclustersFollowBasedTopAuthors, followBasedTopAuthors ) } } private def getTopAuthorsWithScores( embeddings: Map[String, Double] ): Future[Seq[(Long, Double)]] = { val flattenedAuthorsWithScoresFut = Future .collect { embeddings.map { case (clusterId, seedScore) => getTopAuthorsWithScoresForCluster(clusterId).map { topAuthors => topAuthors.map { case (author, score) => (author, score * seedScore) } } }.toSeq }.map(_.flatten) flattenedAuthorsWithScoresFut.map { flattenedAuthorsWithScores => val authorsWithScores = flattenedAuthorsWithScores.groupBy(_._1).mapValues(_.map(_._2).sum).toSeq authorsWithScores.sortBy(-_._2) } } private def getTopAuthorsWithScoresForCluster( clusterId: String ): Future[Seq[(Long, Double)]] = { cache .get(clusterId) .map(Future.value(_)) .getOrElse { store .get(clusterId).map { clusterDetailsOpt => val authorsWithScores = clusterDetailsOpt .flatMap { clusterDetails => clusterDetails.knownForUsersAndScores.map { knownForUsersAndScores => knownForUsersAndScores.map { userAndScore => (userAndScore.userId, userAndScore.score) } } }.getOrElse(Seq.empty) cache.set(clusterId, authorsWithScores) authorsWithScores }.handle { case _ => Seq.empty } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SlopAuthorFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.abuse.detection.scoring.{thriftscala => t} import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature import com.twitter.home_mixer.model.HomeFeatures.SlopAuthorFeature import com.twitter.home_mixer.model.HomeFeatures.SlopAuthorScoreFeature import com.twitter.home_mixer.param.HomeGlobalParams.SlopMaxScore import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.hss.user_scores.api.HealthModelScoresOnUserClientColumn import javax.inject.Inject import javax.inject.Singleton @Singleton class SlopAuthorFeatureHydrator @Inject() ( healthModelScoresOnUserClientColumn: HealthModelScoresOnUserClientColumn) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("SlopAuthor") override val features: Set[Feature[_, _]] = Set(SlopAuthorFeature, DebugStringFeature, SlopAuthorScoreFeature) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)).distinct val slopThreshold = query.params(SlopMaxScore) Stitch .collectToTry { authorIds.map { authorId => healthModelScoresOnUserClientColumn.fetcher .fetch(authorId, Seq(t.UserHealthModel.NsfwConsumerFollowerScore)) .map { response => authorId -> response.v.flatMap { scoresMap => scoresMap.get(t.UserHealthModel.NsfwConsumerFollowerScore) } } } }.map { authorNsfwScores => val authorIdsToNsfwScoresMap = authorNsfwScores.flatMap(_.toOption).toMap candidates.map { candidate => val debugStringFeature = candidate.features.getOrElse(DebugStringFeature, None).getOrElse("") candidate.features.getOrElse(AuthorIdFeature, None) match { case Some(authorId) => val slopAuthorScore = authorIdsToNsfwScoresMap.getOrElse(authorId, None).getOrElse(0.0) FeatureMap( SlopAuthorFeature, slopAuthorScore > slopThreshold, DebugStringFeature, Some("%s Slop %.3f".format(debugStringFeature, slopAuthorScore)), SlopAuthorScoreFeature, Some(slopAuthorScore) ) case _ => FeatureMap( SlopAuthorFeature, false, DebugStringFeature, Some(debugStringFeature), SlopAuthorScoreFeature, None ) } } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SpaceStateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature import com.twitter.periscope.api.{thriftscala => ps} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.strato.catalog.Fetch import com.twitter.strato.client.Fetcher import com.twitter.strato.generated.client.periscope.CoreOnAudioSpaceClientColumn import com.twitter.ubs.{thriftscala => ubs} import javax.inject.Inject import javax.inject.Singleton object SpaceStateFeature extends Feature[TweetCandidate, Option[ubs.BroadcastState]] @Singleton class SpaceStateFeatureHydrator @Inject() ( coreOnAudioSpaceClientColumn: CoreOnAudioSpaceClientColumn) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with WithDefaultFeatureMap { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("SpaceState") override val features: Set[Feature[_, _]] = Set(SpaceStateFeature) private val pattern = """https?://(?:x|twitter).com/i/spaces/(\w+).*""".r private val fetcher: Fetcher[String, ps.AudioSpacesLookupContext, ubs.AudioSpace] = coreOnAudioSpaceClientColumn.fetcher private val lookupContext = ps.AudioSpacesLookupContext(participantHydrationLevel = Some(ps.ParticipantHydrationLevel.NoParticipantInfo)) override val defaultFeatureMap: FeatureMap = FeatureMap(SpaceStateFeature, None) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { val spaceIdMap = candidates.flatMap { candidate => candidate.features .getOrElse(TweetUrlsFeature, Seq.empty) .collectFirst { case pattern(spaceId) => spaceId } .map(spaceId => candidate.candidate.id -> spaceId) }.toMap Stitch .collect { spaceIdMap.values.toSeq.distinct.map { spaceId => fetcher .fetch(spaceId, lookupContext) .map { case Fetch.Result(Some(audioSpace), _) if audioSpace.broadcastId.nonEmpty => Some(spaceId -> audioSpace) case _ => None } } }.map { results => val audioSpaceMap = results.flatten.toMap candidates.map { candidate => val spaceState = spaceIdMap.get(candidate.candidate.id).flatMap { spaceId => audioSpaceMap.get(spaceId).flatMap(_.state) } FeatureMapBuilder().add(SpaceStateFeature, spaceState).build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TSPInferredTopicFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.contentrecommender.{thriftscala => cr} import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.clients.strato.topics.TopicSocialProofClient import com.twitter.topiclisting.TopicListingViewerContext import com.twitter.tsp.thriftscala.TopicFollowType import com.twitter.tsp.{thriftscala => tsp} import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]] object TSPInferredTopicDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TSPInferredTopicFeatureHydrator @Inject() ( topicSocialProofClient: TopicSocialProofClient) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic") override val features: Set[Feature[_, _]] = Set( TSPInferredTopicFeature, TSPInferredTopicDataRecordFeature, TopicIdSocialContextFeature, TopicContextFunctionalityTypeFeature ) private val topK = 3 private val SuggestTypesToSetSocialProof: Set[hmt.ServedType] = Set( hmt.ServedType.ForYouTweetMixer, hmt.ServedType.ForYouSimclusters, hmt.ServedType.ForYouTwhin, hmt.ServedType.ForYouUtg, hmt.ServedType.ForYouUvg, hmt.ServedType.ForYouUteg, hmt.ServedType.ForYouPopularGeo, hmt.ServedType.ForYouPopularTopic, hmt.ServedType.ForYouDeepRetrieval, hmt.ServedType.ForYouEvergreenDeepRetrieval, hmt.ServedType.ForYouRelatedCreator, hmt.ServedType.ForYouLocal, hmt.ServedType.ForYouTrends, hmt.ServedType.ForYouHistoryAuthor, ) private val DefaultFeatureMap = FeatureMap( TSPInferredTopicFeature, Map.empty[Long, Double], TSPInferredTopicDataRecordFeature, new DataRecord(), TopicIdSocialContextFeature, None, TopicContextFunctionalityTypeFeature, None ) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val tags = candidates.collect { case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn => candidate.candidate.id -> candidate.features .getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag]) }.toMap val followableUttTopics = query.features .flatMap(_.getOrElse(FollowableUttTopicsFeatures, None)) .map(_.mapValues(_.getOrElse(TopicFollowType.ImplicitFollow))) val topicSocialProofRequest = tsp.TopicSocialProofRequest( userId = query.getRequiredUserId, tweetIds = candidates.map(_.candidate.id).toSet, displayLocation = cr.DisplayLocation.HomeTimeline, topicListingSetting = tsp.TopicListingSetting.Followable, context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift, bypassModes = None, allowlist = followableUttTopics, // Only TweetMixer source has this data. Convert the TweetMixer metric tag to tsp metric tag. tags = if (tags.isEmpty) None else Some(tags) ) topicSocialProofClient .getTopicTweetSocialProofResponse(topicSocialProofRequest) .map { case Some(response) => handleResponse(response, candidates) case _ => candidates.map { _ => DefaultFeatureMap } } } private def handleResponse( response: tsp.TopicSocialProofResponse, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[FeatureMap] = { candidates.map { candidate => val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty) if (topicWithScores.nonEmpty) { val (socialProofId, socialProofFunctionalityType) = if (SuggestTypesToSetSocialProof.contains( candidate.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined))) { getSocialProof(topicWithScores) } else (None, None) val inferredTopicFeatures = topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap val inferredTopicDataRecord = InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head FeatureMap( TSPInferredTopicFeature, inferredTopicFeatures, TSPInferredTopicDataRecordFeature, inferredTopicDataRecord, TopicIdSocialContextFeature, socialProofId, TopicContextFunctionalityTypeFeature, socialProofFunctionalityType ) } else DefaultFeatureMap } } private def getSocialProof( topicWithScores: Seq[tsp.TopicWithScore] ): (Option[Long], Option[TopicContextFunctionalityType]) = { val followingTopicId = topicWithScores.collectFirst { case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) => topicId } if (followingTopicId.nonEmpty) { return (followingTopicId, Some(BasicTopicContextFunctionalityType)) } val implicitFollowingId = topicWithScores.collectFirst { case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) => topicId } if (implicitFollowingId.nonEmpty) { return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType)) } (None, None) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TransformerPostEmbeddingFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.PostTransformerEmbeddingsHomeBlueAdapter import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.PostTransformerEmbeddingsHomeGreenAdapter import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.PostTransformerEmbeddingsJointBlueAdapter import com.twitter.home_mixer_features.{thriftscala => hmf} import com.twitter.ml.api.DataRecord import com.twitter.ml.api.{thriftscala => ml} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ object TransformerPostEmbeddingHomeBlueFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object TransformerPostEmbeddingHomeGreenFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object TransformerPostEmbeddingJointBlueFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TransformerPostEmbeddingHomeBlueFeatureHydrator @Inject() ( homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, statsReceiver: StatsReceiver) extends TransformerPostEmbeddingFeatureHydrator( homeMixerFeatureService, statsReceiver, hmf.Cache.TransformerPostEmbeddings, TransformerPostEmbeddingHomeBlueFeature, PostTransformerEmbeddingsHomeBlueAdapter ) { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "TransformerPostEmbeddingBlue") } @Singleton class TransformerPostEmbeddingHomeGreenFeatureHydrator @Inject() ( homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, statsReceiver: StatsReceiver) extends TransformerPostEmbeddingFeatureHydrator( homeMixerFeatureService, statsReceiver, hmf.Cache.TransformerPostEmbeddingsGreen, TransformerPostEmbeddingHomeGreenFeature, PostTransformerEmbeddingsHomeGreenAdapter ) { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "TransformerPostEmbeddingGreen") } @Singleton class TransformerPostEmbeddingJointBlueFeatureHydrator @Inject() ( homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, statsReceiver: StatsReceiver) extends TransformerPostEmbeddingFeatureHydrator( homeMixerFeatureService, statsReceiver, hmf.Cache.TransformerPostJointEmbeddingsBlue, TransformerPostEmbeddingJointBlueFeature, PostTransformerEmbeddingsJointBlueAdapter ) { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "TransformerPostEmbeddingGreen") } abstract class TransformerPostEmbeddingFeatureHydrator( homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, statsReceiver: StatsReceiver, cache: hmf.Cache, feature: DataRecordInAFeature[TweetCandidate], dataRecordAdapter: TimelinesMutatingAdapterBase[Option[ml.FloatTensor]]) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val features: Set[Feature[_, _]] = Set(feature) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyTotalCounter = scopedStatsReceiver.counter("key/total") private val batchSize = 50 private def getEmbeddingsFromHMF( tweetIds: Seq[Long] ): Future[Seq[Option[Seq[Double]]]] = { val keysSerialized = tweetIds.map(_.toString) val request = hmf.HomeMixerFeaturesRequest(keysSerialized, cache) val responseFut = homeMixerFeatureService.getHomeMixerFeatures(request) responseFut .map { response => response.homeMixerFeatures .map { homeMixerFeatureOpt => homeMixerFeatureOpt.homeMixerFeaturesType.map { case hmf.HomeMixerFeaturesType.RawEmbedding(rawEmbedding) => rawEmbedding case _ => throw new Exception("Unknown type returned") } } }.handle { case _ => Seq.fill(tweetIds.size)(None) } } private def getBatchedFeatureMap( candidatesBatch: Seq[CandidateWithFeatures[TweetCandidate]] ): Future[Seq[FeatureMap]] = { val tweetIds = candidatesBatch.map { candidate => keyTotalCounter.incr() candidate.candidate.id } getEmbeddingsFromHMF(tweetIds).map { response => response.map { embeddingOpt => val floatTensor = embeddingOpt.map { embedding => keyFoundCounter.incr() ml.FloatTensor(embedding) } val dataRecord = dataRecordAdapter.adaptToDataRecords(floatTensor).asScala.head FeatureMap(feature, dataRecord) } } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { OffloadFuturePools.offloadBatchSeqToFutureSeq(candidates, getBatchedFeatureMap, batchSize) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetEntityServiceContentFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.knuddels.jtokkit.Encodings import com.knuddels.jtokkit.api.Encoding import com.knuddels.jtokkit.api.ModelType import com.twitter.escherbird.{thriftscala => esb} import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content.ContentFeatureAdapter import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature import com.twitter.home_mixer.model.HomeFeatures.HasMultipleMedia import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.IsSelfThreadFeature import com.twitter.home_mixer.model.HomeFeatures.MediaCategoryFeature import com.twitter.home_mixer.model.HomeFeatures.MediaIdFeature import com.twitter.home_mixer.model.HomeFeatures.MediaUnderstandingAnnotationIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SemanticAnnotationIdsFeature import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFromTweetypieFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextTokensFeature import com.twitter.home_mixer.model.HomeFeatures.VideoAspectRatioFeature import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature import com.twitter.home_mixer.model.HomeFeatures.VideoHeightFeature import com.twitter.home_mixer.model.HomeFeatures.VideoWidthFeature import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetypieContentFeaturesDeciderParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetypieContentFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetypieContentMediaEntityFeaturesParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableContentFeatureFromTesService import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.home_mixer.util.tweetypie.content.FeatureExtractionHelper import com.twitter.home_mixer_features.{thriftscala => hmf} import com.twitter.mediaservices.commons.thriftscala.MediaCategory import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.common.util.MediaUnderstandingAnnotations import com.twitter.tweetypie.{thriftscala => tp} import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ object TweetypieContentDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } case class TweetContentExtractionResult( annotations: Seq[esb.TweetEntityAnnotation] = Seq.empty, contentDataRecord: DataRecord = new DataRecord(), videoDurationMs: Option[Int] = None, tweetLanguage: Option[String] = None, tweetText: Option[String] = None, tweetTextTokens: Option[Seq[Int]] = None, aspectRatio: Option[Float] = None, height: Option[Short] = None, width: Option[Short] = None, isSelfThread: Boolean = false, mediaId: Option[Long] = None, mediaCategory: Option[MediaCategory] = None, hasMultipleMedia: Option[Boolean] = None, hasImage: Option[Boolean] = None, hasVideo: Option[Boolean] = None) @Singleton class TweetEntityServiceContentFeatureHydrator @Inject() ( homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, override val statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with ObservedKeyValueResultHandler with Conditionally[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "TweetEntityServiceContent") override val features: Set[Feature[_, _]] = Set( MediaUnderstandingAnnotationIdsFeature, SemanticAnnotationIdsFeature, TweetypieContentDataRecordFeature, VideoDurationMsFeature, TweetLanguageFromTweetypieFeature, TweetTextFeature, TweetTextTokensFeature, VideoAspectRatioFeature, VideoHeightFeature, VideoWidthFeature, IsSelfThreadFeature, MediaIdFeature, MediaCategoryFeature, HasMultipleMedia, HasImageFeature, HasVideoFeature ) override def onlyIf( query: PipelineQuery ): Boolean = query.params(EnableTweetypieContentFeaturesDeciderParam) && query.params(EnableTweetypieContentFeaturesParam) override val statScope: String = identifier.toString private val batchSize = 64 val tokenizer: Encoding = Encodings.newLazyEncodingRegistry().getEncodingForModel(ModelType.GPT_4) private def getContentFeaturesFromHMF( tweetIdsToHydrate: Seq[Long], getFromTES: Boolean = false ): Future[Seq[Option[tp.Tweet]]] = { val keysSerialized = tweetIdsToHydrate.map(_.toString) val request = hmf.HomeMixerFeaturesRequest( keysSerialized, hmf.Cache.TweetypieContent, Some( hmf.HomeMixerFeaturesRequestContext.ContentFeatureRequestContext( hmf.ContentFeatureRequestContext(Some(getFromTES)) )) ) val responseFut = homeMixerFeatureService.getHomeMixerFeatures(request) responseFut .map { response => response.homeMixerFeatures .map { homeMixerFeatureOpt => homeMixerFeatureOpt.homeMixerFeaturesType.map { case hmf.HomeMixerFeaturesType.TweetypieContent(homeMixerFeature) => homeMixerFeature case _ => throw new Exception("Unknown type returned") } } }.handle { case _ => Seq.fill(tweetIdsToHydrate.size)(None) } } private def getFeatureMaps( candidates: Seq[CandidateWithFeatures[TweetCandidate]], query: PipelineQuery, getFromTes: Boolean ): Future[Seq[FeatureMap]] = { val tweetIdsToHydrate = candidates.map(CandidatesUtil.getOriginalTweetId) val isExtractMediaEntities = query.params(EnableTweetypieContentMediaEntityFeaturesParam) val responseMap = getContentFeaturesFromHMF(tweetIdsToHydrate, getFromTes) responseMap.map { result => result.map { tweetContent => val transformed = postTransformer(tweetContent, isExtractMediaEntities) val annotationIds = transformed.annotations.map(_.entityId) val mediaUnderstandingAnnotationIds = getNonSensitiveHighRecallMediaUnderstandingAnnotationEntityIds(transformed.annotations) FeatureMapBuilder(sizeHint = 13) .add(MediaUnderstandingAnnotationIdsFeature, mediaUnderstandingAnnotationIds) .add(SemanticAnnotationIdsFeature, annotationIds) .add(TweetypieContentDataRecordFeature, transformed.contentDataRecord) .add(VideoDurationMsFeature, transformed.videoDurationMs) .add(TweetLanguageFromTweetypieFeature, transformed.tweetLanguage) .add(TweetTextFeature, transformed.tweetText) .add(TweetTextTokensFeature, transformed.tweetTextTokens) .add(VideoAspectRatioFeature, transformed.aspectRatio) .add(VideoHeightFeature, transformed.height) .add(VideoWidthFeature, transformed.width) .add(IsSelfThreadFeature, transformed.isSelfThread) .add(MediaIdFeature, transformed.mediaId) .add(MediaCategoryFeature, transformed.mediaCategory) .add(HasMultipleMedia, transformed.hasMultipleMedia.getOrElse(false)) .add(HasImageFeature, transformed.hasImage.getOrElse(false)) .add(HasVideoFeature, transformed.hasVideo.getOrElse(false)) .build() } } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val getFromTes = query.params(EnableContentFeatureFromTesService) OffloadFuturePools.offloadBatchSeqToFutureSeq( candidates, getFeatureMaps(_, query, getFromTes), batchSize) } private def postTransformer( result: Option[tp.Tweet], isExtractMediaEntities: Boolean = true ): TweetContentExtractionResult = { val transformedValue = result.map(FeatureExtractionHelper.extractFeatures(_, isExtractMediaEntities)) val semanticAnnotations = transformedValue.flatMap { _.semanticCoreAnnotations }.getOrElse(Seq.empty) val dataRecord = ContentFeatureAdapter.adaptToDataRecords(transformedValue).asScala.head val videoDurationMs = transformedValue.flatMap { _.videoDurationMs } val mediaId = transformedValue.flatMap { _.media.flatMap(_.headOption).map(_.mediaId) } val hasMultipleMedia = Some(transformedValue.map(_.media.map(_.size > 1).getOrElse(false)).getOrElse(false)) val mediaCategory = transformedValue.flatMap { _.media.flatMap(_.headOption).flatMap(_.mediaKey).map(_.mediaCategory) } val tweetLanguage = result.flatMap { _.language.map(_.language) } val tweetText = result.flatMap { _.coreData.map(_.text) } val tweetTextTokens = tweetText.map { text => tokenizer.encodeOrdinary(text, 1024).getTokens.toArray.toSeq } val aspectRatioNum = transformedValue.flatMap { _.aspectRatioNum } val aspectRatioDen = transformedValue.flatMap { _.aspectRatioDen } val aspectRatio = aspectRatioNum .zip(aspectRatioDen).map { case (num, den) => if (den != 0) num.toFloat / den.toFloat else -1 }.find(_ > 0) val mediaHeight = transformedValue.flatMap { _.heights.flatMap(_.headOption) } val mediaWidth = transformedValue.flatMap { _.widths.flatMap(_.headOption) } val isSelfThread = transformedValue.exists(_.selfThreadMetadata.nonEmpty) val hasImage = transformedValue.flatMap(_.hasImage) val hasVideo = transformedValue.flatMap(_.hasVideo) TweetContentExtractionResult( semanticAnnotations, dataRecord, videoDurationMs, tweetLanguage, tweetText, tweetTextTokens, aspectRatio, mediaHeight, mediaWidth, isSelfThread, mediaId, mediaCategory, hasMultipleMedia, hasImage, hasVideo ) } private def getNonSensitiveHighRecallMediaUnderstandingAnnotationEntityIds( semanticCoreAnnotations: Seq[esb.TweetEntityAnnotation] ): Seq[Long] = semanticCoreAnnotations .filter(MediaUnderstandingAnnotations.isEligibleNonSensitiveHighRecallMUAnnotation) .map(_.entityId) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetEntityServiceFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.conversions.DurationOps.RichDuration import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FirstMediaIdFeature import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsArticleFeature import com.twitter.home_mixer.model.HomeFeatures.IsInReplyToReplyOrDirectedFeature import com.twitter.home_mixer.model.HomeFeatures.IsInReplyToRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.OriginalTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.TesBatchedStratoClient import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieStaticEntitiesCache import com.twitter.home_mixer.util.tweetypie.content.TweetMediaFeaturesExtractor import com.twitter.mediaservices.commons.thriftscala.MediaKey import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationIdFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.cache.TtlCache import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.stitch.Arrow import com.twitter.stitch.Stitch import com.twitter.strato.client.Client import com.twitter.strato.client.CommunityId import com.twitter.strato.client.UserId import com.twitter.strato.generated.client.tweetypie.federated.ArticleOnTweetClientColumn import com.twitter.strato.generated.client.tweetypie.federated.CommunityOnTweetClientColumn import com.twitter.strato.generated.client.tweetypie.federated.ContextualQuoteTweetRefOnTweetClientColumn import com.twitter.strato.generated.client.tweetypie.federated.DirectedAtUserOnTweetClientColumn import com.twitter.strato.generated.client.tweetypie.federated.ExclusiveTweetControlOnTweetClientColumn import com.twitter.strato.generated.client.tweetypie.federated.MediaKeysOnTweetClientColumn import com.twitter.strato.generated.client.tweetypie.federated.MentionsOnTweetClientColumn import com.twitter.strato.generated.client.tweetypie.federated.NarrowcastPlaceOnTweetClientColumn import com.twitter.strato.generated.client.tweetypie.federated.PureCoreDataOnTweetClientColumn import com.twitter.strato.generated.client.tweetypie.federated.UrlsOnTweetClientColumn import com.twitter.strato.graphql.contextual_refs.thriftscala.ContextualTweetRef import com.twitter.tweetypie.{thriftscala => tp} import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class TweetEntityServiceFeatureHydrator @Inject() ( @Named(TweetypieStaticEntitiesCache) cacheClient: TtlCache[Long, tp.Tweet], @Named(TesBatchedStratoClient) stratoClient: Client, statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { import TweetEntityServiceFeatureHydrator._ override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(IdentifierName) override val features: Set[Feature[_, _]] = Features private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val pureCoreDataNonEmptyCounter = scopedStatsReceiver.counter("pureCoreDataNonEmpty") private val pureCoreDataEmptyCounter = scopedStatsReceiver.counter("pureCoreDataEmpty") private lazy val getTESRecord: Arrow[Long, TESRecord] = getTESRecordArrow(stratoClient) private lazy val getFromTES: Arrow[Seq[Long], Map[Long, tp.Tweet]] = Arrow .sequence(getTESRecord) .map { record => record .map(_.getTweetOpt).filter { tweetOpt => if (tweetOpt.nonEmpty) pureCoreDataNonEmptyCounter.incr() else pureCoreDataEmptyCounter.incr() tweetOpt.nonEmpty }.map(tweet => (tweet.get.id, tweet.get)).toMap } private lazy val getHydratedTweetMapWithCacheWriteBack: Arrow[ KeyValueResult[Long, tp.Tweet], Map[Long, tp.Tweet] ] = Arrow .zipWithArg( Arrow .identity[KeyValueResult[Long, tp.Tweet]] .andThen(getFromTES.contramap[KeyValueResult[Long, tp.Tweet]](kv => kv.notFound.toSeq)) ).map { case (fromCache, fromTES) => fromTES.map(kv => cacheClient.set(kv._1, kv._2, CacheTTL)) fromCache.found ++ fromTES } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { val tweetIds: Seq[Long] = candidates.map(_.candidate.id) val inReplyToIds = candidates.flatMap(_.features.getOrElse(InReplyToTweetIdFeature, None)) val idsToHydrate = (tweetIds ++ inReplyToIds).distinct Stitch .callFuture(cacheClient.get(idsToHydrate)) .flatMap(cacheResult => getHydratedTweetMapWithCacheWriteBack(cacheResult)) .map { tweetMap: Map[Long, tp.Tweet] => candidates.map { candidate => tweetMap .get(candidate.candidate.id).map { tweet => getFeatureMapFromTweet( tweet, candidate.features.getOrElse(InReplyToTweetIdFeature, None).flatMap(tweetMap.get) ) }.getOrElse(DefaultFeatureMap) } } } } object TweetEntityServiceFeatureHydrator { private val IdentifierName = "TweetEntityService" private val CacheTTL = 48.hours private val Features: Set[Feature[_, _]] = Set( AuthorIdFeature, CommunityIdFeature, DirectedAtUserIdFeature, ExclusiveConversationAuthorIdFeature, FirstMediaIdFeature, HasImageFeature, HasVideoFeature, IsArticleFeature, InReplyToTweetIdFeature, InReplyToUserIdFeature, IsInReplyToReplyOrDirectedFeature, IsInReplyToRetweetFeature, IsRetweetFeature, LocationIdFeature, MentionScreenNameFeature, MentionUserIdFeature, QuotedTweetIdFeature, OriginalTweetIdFeature, SourceTweetIdFeature, SourceUserIdFeature, TweetMediaIdsFeature, TweetUrlsFeature, ) private val DefaultFeatureMap = FeatureMapBuilder() .add(AuthorIdFeature, None) .add(DirectedAtUserIdFeature, None) .add(ExclusiveConversationAuthorIdFeature, None) .add(HasImageFeature, false) .add(HasVideoFeature, false) .add(IsArticleFeature, false) .add(InReplyToTweetIdFeature, None) .add(InReplyToUserIdFeature, None) .add(IsInReplyToReplyOrDirectedFeature, false) .add(IsInReplyToRetweetFeature, false) .add(IsRetweetFeature, false) .add(LocationIdFeature, None) .add(MentionScreenNameFeature, Seq.empty) .add(MentionUserIdFeature, Seq.empty) .add(QuotedTweetIdFeature, None) .add(OriginalTweetIdFeature, None) .add(SourceTweetIdFeature, None) .add(SourceUserIdFeature, None) .add(CommunityIdFeature, None) .add(TweetMediaIdsFeature, Seq.empty) .add(FirstMediaIdFeature, None) .add(TweetUrlsFeature, Seq.empty) .build() private def getFeatureMapFromTweet( tweet: tp.Tweet, inReplyToTweet: Option[tp.Tweet] ): FeatureMap = { val coreData = tweet.coreData val quotedTweet = tweet.quotedTweet val mentions = tweet.mentions.getOrElse(Seq.empty) val share = coreData.flatMap(_.share) val reply = coreData.flatMap(_.reply) val urls = tweet.urls.map(_.flatMap(_.expanded)).toSeq.flatten val (isInReplyToReplyOrDirected, isInReplyToRetweet) = inReplyToTweet .map { tweet => ( // when inReplyToUserId exists, it can be a reply or a directedAt tweet, // depending on whether inReplyToTweetId exists tweet.coreData.flatMap(_.reply).map(_.inReplyToUserId).isDefined, tweet.coreData.flatMap(_.share).isDefined ) }.getOrElse((false, false)) // There are cases where the inReplyToUserId exists while inReplyToStatusId does not. // They're usually directed tweets that are not replies. val inReplyToTweetId = reply.flatMap(_.inReplyToStatusId) val inReplyToUserId = if (inReplyToTweetId.nonEmpty) reply.map(_.inReplyToUserId) else None val tweetMediaIds = TweetMediaFeaturesExtractor.getMediaIds(tweet) FeatureMapBuilder() .add(AuthorIdFeature, coreData.map(_.userId)) .add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId))) .add( ExclusiveConversationAuthorIdFeature, tweet.exclusiveTweetControl.map(_.conversationAuthorId)) .add(HasImageFeature, TweetMediaFeaturesExtractor.hasImage(tweet)) .add(HasVideoFeature, TweetMediaFeaturesExtractor.hasVideo(tweet)) .add(IsArticleFeature, tweet.article.isDefined) .add(InReplyToTweetIdFeature, inReplyToTweetId) .add(InReplyToUserIdFeature, inReplyToUserId) .add(IsRetweetFeature, share.isDefined) .add(IsInReplyToReplyOrDirectedFeature, isInReplyToReplyOrDirected) .add(IsInReplyToRetweetFeature, isInReplyToRetweet) .add(LocationIdFeature, tweet.narrowcastPlace.map(_.id)) .add(MentionScreenNameFeature, mentions.map(_.screenName)) .add(MentionUserIdFeature, mentions.flatMap(_.userId)) .add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId)) .add(OriginalTweetIdFeature, Some(share.map(_.sourceStatusId).getOrElse(tweet.id))) .add(SourceTweetIdFeature, share.map(_.sourceStatusId)) .add(SourceUserIdFeature, share.map(_.sourceUserId)) .add(CommunityIdFeature, tweet.communities.flatMap(_.communityIds.headOption)) .add(TweetMediaIdsFeature, tweetMediaIds) .add(FirstMediaIdFeature, tweetMediaIds.headOption) .add(TweetUrlsFeature, urls) .build() } private def getTESRecordArrow(stratoClient: Client): Arrow[Long, TESRecord] = { val pureCoreDataArrow: Arrow[Long, Option[tp.PureCoreData]] = new PureCoreDataOnTweetClientColumn(stratoClient).fetcher.asArrow .contramap[Long](tweetId => (tweetId, ())) .map(_.v) val communityArrow: Arrow[Long, Option[CommunityId]] = new CommunityOnTweetClientColumn(stratoClient).fetcher.asArrow .contramap[Long](tweetId => (tweetId, ())) .map(_.v) val directedAtUserArrow: Arrow[Long, Option[UserId]] = new DirectedAtUserOnTweetClientColumn(stratoClient).fetcher.asArrow .contramap[Long](tweetId => (tweetId, ())) .map(_.v) val exclusiveTweetControlArrow: Arrow[Long, Option[tp.ExclusiveTweetControl]] = new ExclusiveTweetControlOnTweetClientColumn(stratoClient).fetcher.asArrow .contramap[Long](tweetId => (tweetId, ())) .map(_.v) val mediaKeysArrow: Arrow[Long, Option[Seq[MediaKey]]] = new MediaKeysOnTweetClientColumn(stratoClient).fetcher.asArrow .contramap[Long](tweetId => (tweetId, ())) .map(_.v) val mentionsArrow: Arrow[Long, Option[Seq[tp.MentionEntity]]] = new MentionsOnTweetClientColumn(stratoClient).fetcher.asArrow .contramap[Long](tweetId => (tweetId, ())) .map(_.v) val contextualQuoteTweetRefArrow: Arrow[Long, Option[ContextualTweetRef]] = new ContextualQuoteTweetRefOnTweetClientColumn(stratoClient).fetcher.asArrow .contramap[Long](tweetId => (tweetId, ())) .map(_.v) val narrowcastPlaceArrow: Arrow[Long, Option[tp.NarrowcastPlace]] = new NarrowcastPlaceOnTweetClientColumn(stratoClient).fetcher.asArrow .contramap[Long](tweetId => (tweetId, ())) .map(_.v) val articleArrow: Arrow[Long, Option[tp.Article]] = new ArticleOnTweetClientColumn(stratoClient).fetcher.asArrow .contramap[Long](tweetId => (tweetId, ())) .map(_.v) val urlsArrow: Arrow[Long, Option[Seq[tp.UrlEntity]]] = new UrlsOnTweetClientColumn(stratoClient).fetcher.asArrow .contramap[Long](tweetId => (tweetId, ())) .map(_.v) Arrow .zipWithArg( Arrow .identity[Long].andThen(Arrow.join( pureCoreDataArrow, articleArrow, communityArrow, directedAtUserArrow, exclusiveTweetControlArrow, mediaKeysArrow, mentionsArrow, narrowcastPlaceArrow, contextualQuoteTweetRefArrow, urlsArrow )) ).map { case ( tweetId, ( pureCoreDataOpt: Option[tp.PureCoreData], articleOpt: Option[tp.Article], communityIdOpt: Option[CommunityId], directedAtUserIdOpt: Option[UserId], exclusiveTweetControls: Option[tp.ExclusiveTweetControl], mediaKeysOpt: Option[Seq[MediaKey]], mentionEntitiesOpt: Option[Seq[tp.MentionEntity]], narrowcastPlaceOpt: Option[tp.NarrowcastPlace], quotedTweetOpt: Option[ContextualTweetRef], urlsOpt: Option[Seq[tp.UrlEntity]] )) => TESRecord( tweetId, pureCoreDataOpt, articleOpt, communityIdOpt, directedAtUserIdOpt, exclusiveTweetControls, mediaKeysOpt, mentionEntitiesOpt, narrowcastPlaceOpt, quotedTweetOpt, urlsOpt ) } } } case class TESRecord( tweetId: Long, pureCoreDataOpt: Option[tp.PureCoreData], articleOpt: Option[tp.Article], communityIdOpt: Option[CommunityId], directedAtUserIdOpt: Option[UserId], exclusiveTweetControls: Option[tp.ExclusiveTweetControl], mediaKeysOpt: Option[Seq[MediaKey]], mentionEntitiesOpt: Option[Seq[tp.MentionEntity]], narrowcastPlaceOpt: Option[tp.NarrowcastPlace], quotedTweetOpt: Option[ContextualTweetRef], urlsOpt: Option[Seq[tp.UrlEntity]]) { def getTweetOpt: Option[tp.Tweet] = pureCoreDataOpt.map { pureCoreData => tp.Tweet( id = tweetId, coreData = Some( tp.TweetCoreData.unsafeEmpty.copy( userId = pureCoreData.userId, share = pureCoreData.share, reply = pureCoreData.reply, directedAtUser = directedAtUserIdOpt.map(id => tp.DirectedAtUser(id.value, "")) )), article = articleOpt, communities = communityIdOpt.map(community => tp.Communities(Seq(community.value))), exclusiveTweetControl = exclusiveTweetControls .map(control => tp.ExclusiveTweetControl(control.conversationAuthorId)), media = mediaKeysOpt.map { mediaKeys => mediaKeys.map { key => tp.MediaEntity.unsafeEmpty.copy(mediaId = key.mediaId, mediaKey = Some(key)) } }, mentions = mentionEntitiesOpt, narrowcastPlace = narrowcastPlaceOpt, quotedTweet = quotedTweetOpt.map(qt => tp.QuotedTweet.unsafeEmpty.copy(tweetId = qt.id)), urls = urlsOpt ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.TweetImpressionsFeature import com.twitter.home_mixer.model.request.HasSeenTweetIds import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.impression.{thriftscala => t} import com.twitter.timelines.impressionstore.store.ManhattanTweetImpressionStoreClient import com.twitter.util.Duration import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton @Singleton case class TweetImpressionsQueryFeatureHydrator[ Query <: PipelineQuery with HasSeenTweetIds] @Inject() ( manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient) extends QueryFeatureHydrator[Query] { private val TweetImpressionTTL = 2.days private val TweetImpressionCap = 5000 override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions") override val features: Set[Feature[_, _]] = Set(TweetImpressionsFeature) override def hydrate(query: Query): Stitch[FeatureMap] = { manhattanTweetImpressionStoreClient.get(query.getRequiredUserId).map { entriesOpt => val entries = entriesOpt.map(_.entries).toSeq.flatten val updatedImpressions = if (query.seenTweetIds.forall(_.isEmpty)) entries else updateTweetImpressions(entries, query.seenTweetIds.get) FeatureMapBuilder().add(TweetImpressionsFeature, updatedImpressions).build() } } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8) ) /** * 1) Check timestamps and remove expired tweets based on [[TweetImpressionTTL]] * 2) Filter duplicates between current tweets and those in the impression store (remove older ones) * 3) Prepend new (Timestamp, Seq[TweetIds]) to the tweets from the impression store * 4) Truncate older tweets if sum of all tweets across timestamps >= [[TweetImpressionCap]], */ private[feature_hydrator] def updateTweetImpressions( tweetImpressionsFromStore: Seq[t.TweetImpressionsEntry], seenIdsFromClient: Seq[Long], currentTime: Long = Time.now.inMilliseconds, tweetImpressionTTL: Duration = TweetImpressionTTL, tweetImpressionCap: Int = TweetImpressionCap, ): Seq[t.TweetImpressionsEntry] = { val seenIdsFromClientSet = seenIdsFromClient.toSet val dedupedTweetImpressionsFromStore: Seq[t.TweetImpressionsEntry] = tweetImpressionsFromStore .collect { case t.TweetImpressionsEntry(ts, tweetIds) if Time.fromMilliseconds(ts).untilNow < tweetImpressionTTL => t.TweetImpressionsEntry(ts, tweetIds.filterNot(seenIdsFromClientSet.contains)) }.filter { _.tweetIds.nonEmpty } val mergedTweetImpressionsEntries = t.TweetImpressionsEntry(currentTime, seenIdsFromClient) +: dedupedTweetImpressionsFromStore val initialTweetImpressionsWithCap = (Seq.empty[t.TweetImpressionsEntry], tweetImpressionCap) val (truncatedTweetImpressionsEntries: Seq[t.TweetImpressionsEntry], _) = mergedTweetImpressionsEntries .foldLeft(initialTweetImpressionsWithCap) { case ( (tweetImpressions: Seq[t.TweetImpressionsEntry], remainingCap), t.TweetImpressionsEntry(ts, tweetIds)) if remainingCap > 0 => ( t.TweetImpressionsEntry(ts, tweetIds.take(remainingCap)) +: tweetImpressions, remainingCap - tweetIds.size) case (tweetImpressionsWithCap, _) => tweetImpressionsWithCap } truncatedTweetImpressionsEntries.reverse } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetLanguageFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFromLanguageSignalFeature import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.language.tweet.LanguageOnTweetClientColumn import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.util.OffloadFuturePools import javax.inject.Inject import javax.inject.Singleton @Singleton class TweetLanguageFeatureHydrator @Inject() ( languageOnTweetClientColumn: LanguageOnTweetClientColumn, statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetLanguage") private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val failedCounter = scopedStatsReceiver.scope(getClass.getSimpleName).counter("failure") private val DefaultFeatureMap = FeatureMapBuilder().add(TweetLanguageFromLanguageSignalFeature, None).build() override def features: Set[Feature[_, _]] = Set(TweetLanguageFromLanguageSignalFeature) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { OffloadFuturePools.offloadStitch { Stitch.collect { candidates.map { candidate => languageOnTweetClientColumn.fetcher .fetch( CandidatesUtil.getOriginalTweetId(candidate), LanguageOnTweetClientColumn .View(true)).map { result => FeatureMapBuilder() .add(TweetLanguageFromLanguageSignalFeature, result.v) .build() }.rescue { case _ => failedCounter.incr() Stitch.value(DefaultFeatureMap) } } } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetLargeEmbeddingsFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.TweetLargeEmbeddingsFeature import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.TweetLargeEmbeddingsKeyFeature import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableLargeEmbeddingsFeatureHydrationParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelNameParam import com.twitter.home_mixer_features.{thriftscala => hmf} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.large_embeddings.HashingFeatureParams import com.twitter.timelines.prediction.adapters.large_embeddings.HomeMixerLargeEmbeddingsFeatureHydrator import com.twitter.timelines.prediction.adapters.large_embeddings.LargeEmbeddingsAdapter import com.twitter.timelines.prediction.adapters.large_embeddings.TweetLargeEmbeddingsAdapter import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton @Singleton class TweetLargeEmbeddingsFeatureHydrator @Inject() ( statsReceiver: StatsReceiver, override val homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] with HomeMixerLargeEmbeddingsFeatureHydrator { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetLargeEmbeddings") override val features: Set[Feature[_, _]] = Set(TweetLargeEmbeddingsFeature, TweetLargeEmbeddingsKeyFeature) override val adapter: LargeEmbeddingsAdapter = TweetLargeEmbeddingsAdapter override val cacheType: hmf.Cache = hmf.Cache.TweetLargeEmbeddings override val scopedStatsReceiver: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableLargeEmbeddingsFeatureHydrationParam) override val modelName2HashingFeatureParams: Map[String, HashingFeatureParams] = Map( "hr_video_prod__v3_realtime" -> HashingFeatureParams( scales = Seq(1487022661L, 1399245971L), biases = Seq(1372992088L, 632996194L), modulus = 2865175829L, bucketSize = 10000000L, ), "hr_video_prod__v2_lembeds" -> HashingFeatureParams( scales = Seq(2516541900L, 2376187492L), biases = Seq(3022238687L, 1571354734L), modulus = 3047336911L, bucketSize = 1000000L, ), "hr_prod__v4_embeds_230M" -> HashingFeatureParams( scales = Seq(2161410491L, 1754358832L), biases = Seq(296686044L, 1959990826L), modulus = 2361375383L, bucketSize = 100000000L, ), "hr_prod__v5_embeds_230M_and_transformer" -> HashingFeatureParams( scales = Seq(2161410491L, 1754358832L), biases = Seq(296686044L, 1959990826L), modulus = 2361375383L, bucketSize = 100000000L, ), "hr_prod__v5_watchtime" -> HashingFeatureParams( scales = Seq(407033648L, 940305868L), biases = Seq(494266171L, 269596788L), modulus = 949146421L, bucketSize = 100000000L, ), "hr_prod__v6_transformer_v2" -> HashingFeatureParams( scales = Seq(2161410491L, 1754358832L), biases = Seq(296686044L, 1959990826L), modulus = 2361375383L, bucketSize = 100000000L, ), "hr_prod__v6_mixed_training" -> HashingFeatureParams( scales = Seq(2161410491L, 1754358832L), biases = Seq(296686044L, 1959990826L), modulus = 2361375383L, bucketSize = 100000000L, ), "hr_prod__v6_transformer_v2_kafka_merge_join" -> HashingFeatureParams( scales = Seq(2161410491L, 1754358832L), biases = Seq(296686044L, 1959990826L), modulus = 2361375383L, bucketSize = 100000000L, ), "hr_prod__v6_transformer_v2_realtime_debias_21apr" -> HashingFeatureParams( scales = Seq(2161410491L, 1754358832L), biases = Seq(296686044L, 1959990826L), modulus = 2361375383L, bucketSize = 100000000L, ), ) // Hashing Features override val defaultHashingFeatureParams: HashingFeatureParams = HashingFeatureParams( scales = Seq(1131302000L, 303023026L), biases = Seq(799473858L, 600426834L), modulus = 3588720353L, bucketSize = 10000000L, ) private val batchSize = 25 private def getBatchedFeatureMap( modelName: String, candidatesBatch: Seq[CandidateWithFeatures[TweetCandidate]], ): Future[Seq[FeatureMap]] = { val tweetIds = candidatesBatch.map { candidate => candidate.candidate.id } getLargeEmbeddings(tweetIds, modelName).map { responses => responses.map { largeEmbeddingResponse => FeatureMapBuilder() .add(TweetLargeEmbeddingsFeature, largeEmbeddingResponse.dataRecord) .add(TweetLargeEmbeddingsKeyFeature, largeEmbeddingResponse.hashedKeys) .build() } } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val modelName = query.params(ModelNameParam) OffloadFuturePools.offloadBatchSeqToFutureSeq( candidates, getBatchedFeatureMap(modelName, _), batchSize ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetMetaDataFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.ml.api.DataRecord import com.twitter.ml.api.RichDataRecord import com.twitter.ml.api.constant.SharedFeatures import com.twitter.ml.api.util.DataRecordConverters._ import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import java.lang.{Long => JLong} object TweetMetaDataDataRecord extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object TweetMetaDataFeatureHydrator extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetMetaData") override def features: Set[Feature[_, _]] = Set(TweetMetaDataDataRecord) private val batchSize = 64 def getFeatureMap(candidate: CandidateWithFeatures[TweetCandidate]): FeatureMap = { val richDataRecord = new RichDataRecord() setFeatures(richDataRecord, candidate.candidate, candidate.features) FeatureMap(TweetMetaDataDataRecord, richDataRecord.getRecord) } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { OffloadFuturePools.offloadBatchElementToElement(candidates, getFeatureMap, batchSize) } private def setFeatures( richDataRecord: RichDataRecord, candidate: TweetCandidate, existingFeatures: FeatureMap ): Unit = { richDataRecord.setFeatureValue[JLong](SharedFeatures.TWEET_ID, candidate.id) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.ORIGINAL_AUTHOR_ID, CandidatesUtil.getOriginalAuthorId(existingFeatures)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetTimeFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.TweetAgeFeature import com.twitter.ml.api.DataRecord import com.twitter.ml.api.util.FDsl._ import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._ import com.twitter.util.Duration import scala.collection.Searching._ object TweetTimeDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object TweetTimeFeatureHydrator extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with WithDefaultFeatureMap { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetTime") override val features: Set[Feature[_, _]] = Set(TweetTimeDataRecordFeature, TweetAgeFeature) override val defaultFeatureMap: FeatureMap = FeatureMap( TweetTimeDataRecordFeature, TweetTimeDataRecordFeature.defaultValue, TweetAgeFeature, None ) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]], ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { val nonPollingTimestampsMs = query.features.get.getOrElse(NonPollingTimesFeature, Seq.empty) candidates.map { candidate => val tweetFeatures = candidate.features.getOrElse(EarlybirdFeature, None) val timeSinceTweetCreation = SnowflakeId.timeFromIdOpt(candidate.candidate.id).map(query.queryTime.since) val timeSinceTweetCreationMs = timeSinceTweetCreation.map(_.inMillis) val timeSinceSourceTweetCreationOpt = candidate.features .getOrElse(SourceTweetIdFeature, None) .flatMap { sourceTweetId => SnowflakeId.timeFromIdOpt(sourceTweetId).map(query.queryTime.since) }.orElse(timeSinceTweetCreation) val lastFavSinceCreationHrs = tweetFeatures.flatMap(_.lastFavSinceCreationHrs).map(_.toDouble) val lastRetweetSinceCreationHrs = tweetFeatures.flatMap(_.lastRetweetSinceCreationHrs).map(_.toDouble) val lastReplySinceCreationHrs = tweetFeatures.flatMap(_.lastReplySinceCreationHrs).map(_.toDouble) val lastQuoteSinceCreationHrs = tweetFeatures.flatMap(_.lastQuoteSinceCreationHrs).map(_.toDouble) val timeSinceLastFavoriteHrs = getTimeSinceLastEngagementHrs(lastFavSinceCreationHrs, timeSinceSourceTweetCreationOpt) val timeSinceLastRetweetHrs = getTimeSinceLastEngagementHrs(lastRetweetSinceCreationHrs, timeSinceSourceTweetCreationOpt) val timeSinceLastReplyHrs = getTimeSinceLastEngagementHrs(lastReplySinceCreationHrs, timeSinceSourceTweetCreationOpt) val timeSinceLastQuoteHrs = getTimeSinceLastEngagementHrs(lastQuoteSinceCreationHrs, timeSinceSourceTweetCreationOpt) val timeSinceLastNonPollingRequest = nonPollingTimestampsMs.headOption.map(query.queryTime.inMillis - _) val nonPollingRequestsSinceTweetCreation = if (nonPollingTimestampsMs.nonEmpty && timeSinceTweetCreationMs.isDefined) { nonPollingTimestampsMs .search(timeSinceTweetCreationMs.get)(Ordering[Long].reverse) .insertionPoint } else 0.0 val tweetAgeRatio = if (timeSinceTweetCreationMs.exists(_ > 0.0) && timeSinceLastNonPollingRequest.isDefined) { timeSinceLastNonPollingRequest.get / timeSinceTweetCreationMs.get.toDouble } else 0.0 val dataRecord = new DataRecord() .setFeatureValue(IS_TWEET_RECYCLED, false) .setFeatureValue(TWEET_AGE_RATIO, tweetAgeRatio) .setFeatureValueFromOption( TIME_SINCE_TWEET_CREATION, timeSinceTweetCreationMs.map(_.toDouble) ) .setFeatureValue( NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, nonPollingRequestsSinceTweetCreation ) .setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, lastFavSinceCreationHrs) .setFeatureValueFromOption(LAST_RETWEET_SINCE_CREATION_HRS, lastRetweetSinceCreationHrs) .setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, lastReplySinceCreationHrs) .setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, lastQuoteSinceCreationHrs) .setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, timeSinceLastFavoriteHrs) .setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, timeSinceLastRetweetHrs) .setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, timeSinceLastReplyHrs) .setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, timeSinceLastQuoteHrs) FeatureMap(TweetTimeDataRecordFeature, dataRecord, TweetAgeFeature, timeSinceTweetCreationMs) } } private def getTimeSinceLastEngagementHrs( lastEngagementTimeSinceCreationHrsOpt: Option[Double], timeSinceTweetCreation: Option[Duration] ): Option[Double] = lastEngagementTimeSinceCreationHrsOpt.flatMap { lastEngagementTimeHrs => timeSinceTweetCreation.map(_.inHours - lastEngagementTimeHrs) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetTypeMetricsFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.functional_component.decorator.builder.HomeTweetTypePredicates import com.twitter.home_mixer.model.HomeFeatures.TweetTypeMetricsFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.suggests.controller_data.Home object TweetTypeMetricsFeatureHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetTypeMetrics") override val features: Set[Feature[_, _]] = Set(TweetTypeMetricsFeature) override def apply( query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap ): Stitch[FeatureMap] = { // Tweet type metrics are already available for cached tweets and shouldn't be overwritten val tweetTypeMetricsFeature = existingFeatures.getOrElse(TweetTypeMetricsFeature, None) val queryFeatures = query.features.getOrElse(FeatureMap.empty) val tweetTypesByteList = if (tweetTypeMetricsFeature.isEmpty) { val bitset = new java.util.BitSet() val trueTweetTypes = HomeTweetTypePredicates.PredicateMap.collect { // Not combining query and candidate features to reduce cost, instead running predicate separately case (predicateName, predicate) if (predicate(existingFeatures) | predicate(queryFeatures)) => predicateName }.toSet Home.TweetTypeIdxMap.collect { case (tweetType, index) if trueTweetTypes.contains(tweetType) => bitset.set(index) } Some(bitset.toByteArray.toList) } else tweetTypeMetricsFeature Stitch.value(FeatureMapBuilder().add(TweetTypeMetricsFeature, tweetTypesByteList).build()) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsArticleFeature import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.ScoredTweetsProduct import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.home_mixer.param.HomeGlobalParams.EnableTweetEntityServiceMigrationParam import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithLongTimeout import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationIdFeature import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.spam.rtf.{thriftscala => rtf} import com.twitter.stitch.Stitch import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} import com.twitter.strato.client.Client import com.twitter.strato.generated.client.tweetypie.managed.HomeMixerOnTweetClientColumn import com.twitter.tweetypie.{thriftscala => tp} import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class TweetypieFeatureHydrator @Inject() ( tweetypieStitchClient: TweetypieStitchClient, statsReceiver: StatsReceiver, @Named(BatchedStratoClientWithLongTimeout) stratoClient: Client) extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Logging { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie") override val features: Set[Feature[_, _]] = Set( AuthorIdFeature, CommunityIdFeature, ExclusiveConversationAuthorIdFeature, InReplyToTweetIdFeature, IsArticleFeature, IsHydratedFeature, IsNsfw, IsRetweetFeature, LocationIdFeature, QuotedTweetDroppedFeature, QuotedTweetIdFeature, QuotedUserIdFeature, SourceTweetIdFeature, SourceUserIdFeature, TweetTextFeature, TweetLanguageFeature, VisibilityReason ) val HydrationFields: Set[tp.TweetInclude] = Set( tp.TweetInclude.TweetFieldId(tp.Tweet.CommunitiesField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.CoreDataField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.SelfThreadMetadataField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.ExclusiveTweetControlField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.IdField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.LanguageField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.QuotedTweetField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.ArticleField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.NarrowcastPlaceField.id) ) private val tweetypieTweetsFoundCounter = statsReceiver.counter("TweetypieTweetsFound") private val tweetypieTweetsNotFoundCounter = statsReceiver.counter("TweetypieTweetsNotFound") private val tesTweetsFoundCounter = statsReceiver.counter("TesTweetsFound") private val tesTweetsNotFoundCounter = statsReceiver.counter("TesTweetsNotFound") private val DefaultFeatureMap = FeatureMapBuilder() .add(CommunityIdFeature, None) .add(IsArticleFeature, false) .add(IsHydratedFeature, false) .add(IsNsfw, None) .add(LocationIdFeature, None) .add(QuotedTweetDroppedFeature, false) .add(TweetTextFeature, None) .add(VisibilityReason, None) .build() private def buildFeatureMap( gtfResult: Stitch[tp.GetTweetFieldsResult], fromTes: Boolean, exclusiveAuthorIdOpt: Option[Long], existingFeatures: FeatureMap ): Stitch[FeatureMap] = { gtfResult.map { case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) => if (fromTes) tesTweetsFoundCounter.incr() else tweetypieTweetsFoundCounter.incr() val coreData = found.tweet.coreData val isNsfwAdmin = coreData.exists(_.nsfwAdmin) val isNsfwUser = coreData.exists(_.nsfwUser) val quotedTweetDropped = quoteOpt.exists { case _: tp.TweetFieldsResultState.Filtered => true case _: tp.TweetFieldsResultState.NotFound => true case _ => false } val quotedTweetIsNsfw = quoteOpt.exists { case quoteTweet: tp.TweetFieldsResultState.Found => quoteTweet.found.tweet.coreData.exists(data => data.nsfwAdmin || data.nsfwUser) case _ => false } val sourceTweetIsNsfw = found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser)) val tweetText = coreData.map(_.text) val tweetLanguage = found.tweet.language.map(_.language) val tweetAuthorId = coreData.map(_.userId) val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId)) val retweetedTweetId = found.retweetedTweet.map(_.id) val quotedTweetId = quoteOpt.flatMap { case quoteTweet: tp.TweetFieldsResultState.Found => Some(quoteTweet.found.tweet.id) case _ => None } val retweetedTweetUserId = found.retweetedTweet.flatMap(_.coreData).map(_.userId) val quotedTweetUserId = quoteOpt.flatMap { case quoteTweet: tp.TweetFieldsResultState.Found => quoteTweet.found.tweet.coreData.map(_.userId) case _ => None } val isNsfw = isNsfwAdmin || isNsfwUser || sourceTweetIsNsfw || quotedTweetIsNsfw val tpExclusiveAuthorIdOpt = found.tweet.exclusiveTweetControl.map(_.conversationAuthorId) val updatedExclusiveAuthorId = tpExclusiveAuthorIdOpt.orElse(exclusiveAuthorIdOpt) val communityId = found.tweet.communities.flatMap(_.communityIds.headOption) FeatureMapBuilder() .add(AuthorIdFeature, tweetAuthorId) .add(CommunityIdFeature, communityId) .add(ExclusiveConversationAuthorIdFeature, updatedExclusiveAuthorId) .add(InReplyToTweetIdFeature, inReplyToTweetId) .add(IsArticleFeature, found.tweet.article.nonEmpty) .add(IsHydratedFeature, true) .add(IsNsfw, Some(isNsfw)) .add(IsRetweetFeature, retweetedTweetId.isDefined) .add(LocationIdFeature, found.tweet.narrowcastPlace.map(_.id)) .add(QuotedTweetDroppedFeature, quotedTweetDropped) .add(QuotedTweetIdFeature, quotedTweetId) .add(QuotedUserIdFeature, quotedTweetUserId) .add(SourceTweetIdFeature, retweetedTweetId) .add(SourceUserIdFeature, retweetedTweetUserId) .add(TweetLanguageFeature, tweetLanguage) .add(TweetTextFeature, tweetText) .add(VisibilityReason, found.suppressReason) .build() case _ => if (fromTes) tesTweetsNotFoundCounter.incr() else tweetypieTweetsNotFoundCounter.incr() DefaultFeatureMap ++ FeatureMapBuilder() .add(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) .add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt) .add(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) .add(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) .add(LocationIdFeature, existingFeatures.getOrElse(LocationIdFeature, None)) .add(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) .add(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) .add(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) .add(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None)) .add(TweetLanguageFeature, existingFeatures.getOrElse(TweetLanguageFeature, None)) .build() } } override def apply( query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap ): Stitch[FeatureMap] = { val safetyLevel = query.product match { case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest case ForYouProduct => val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true) if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown") } val tweetFieldsOptions = tp.GetTweetFieldsOptions( tweetIncludes = HydrationFields, includeRetweetedTweet = true, includeQuotedTweet = true, visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible, safetyLevel = Some(safetyLevel), forUserId = query.getOptionalUserId ) val exclusiveAuthorIdOpt = existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None) if (query.params(EnableTweetEntityServiceMigrationParam)) { val fetcher = new HomeMixerOnTweetClientColumn(stratoClient).fetcher fetcher .fetch( candidate.id, tweetFieldsOptions ).map(_.v).flatMap { case Some(result) => buildFeatureMap(Stitch.value(result), true, exclusiveAuthorIdOpt, existingFeatures) case None => tesTweetsNotFoundCounter.incr() Stitch.value(DefaultFeatureMap) } } else { val gtfResult: Stitch[tp.GetTweetFieldsResult] = tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions) buildFeatureMap(gtfResult, fromTes = false, exclusiveAuthorIdOpt, existingFeatures) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinAuthorFollowFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinAuthorFollowEmbeddingsAdapter import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollowFeatureRepository import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.ml.api.DataRecord import com.twitter.ml.api.{thriftscala => ml} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.repository.KeyValueRepository import com.twitter.servo.repository.KeyValueResult import com.twitter.stitch.Stitch import com.twitter.util.Future import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object TwhinAuthorFollowFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TwhinAuthorFollowFeatureHydrator @Inject() ( @Named(TwhinAuthorFollowFeatureRepository) client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], override val statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TwhinAuthorFollow") override val features: Set[Feature[_, _]] = Set(TwhinAuthorFollowFeature) override val statScope: String = identifier.toString private val emptyDataRecord = new DataRecord() override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val possiblyAuthorIds = extractKeys(candidates) val authorIds = possiblyAuthorIds.flatten.distinct val response: Future[KeyValueResult[Long, DataRecord]] = if (authorIds.isEmpty) Future.value(KeyValueResult.empty) else client(authorIds).map(_.mapFound(postTransformer)) response.map { result => possiblyAuthorIds.map { possiblyAuthorId => val value = observedGet(key = possiblyAuthorId, keyValueResult = result) .map(_.getOrElse(emptyDataRecord)) FeatureMapBuilder().add(TwhinAuthorFollowFeature, value).build() } } } private def postTransformer(embedding: ml.FloatTensor): DataRecord = { TwhinAuthorFollowEmbeddingsAdapter.adaptToDataRecords(Some(embedding)).asScala.head } private def extractKeys( candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[Long]] = { candidates.map { candidate => CandidatesUtil.getOriginalAuthorId(candidate.features) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinRebuildTweetFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinRebuildTweetEmbeddingsAdapter import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinRebuildTweetEmbeddingsStore import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableHomeMixerFeaturesService import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer_features.{thriftscala => hmf} import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.util.Future import com.twitter.ml.api.{thriftscala => ml} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding import com.twitter.simclusters_v2.thriftscala.TwhinEmbeddingDataset import com.twitter.stitch.Stitch import com.twitter.storehaus.ReadableStore import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object TwhinRebuildTweetFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TwhinRebuildTweetFeatureHydrator @Inject() ( homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, @Named(TwhinRebuildTweetEmbeddingsStore) store: ReadableStore[(Long, Long), TwhinTweetEmbedding], statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "TwhinRebuildTweet") override val features: Set[Feature[_, _]] = Set(TwhinRebuildTweetFeature) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyTotalCounter = scopedStatsReceiver.counter("key/total") private val batchSize = 50 private val versionId = TwhinEmbeddingDataset.RefreshedTwhinTweet.value.toLong private def getTwhinEmbeddingsFromHMF( originalTweetIds: Seq[Long] ): Future[Seq[Option[TwhinTweetEmbedding]]] = { val keysSerialized = originalTweetIds.map(_.toString) val request = hmf.HomeMixerFeaturesRequest(keysSerialized, hmf.Cache.TwhinRebuild) val responseFut = homeMixerFeatureService.getHomeMixerFeatures(request) responseFut .map { response => response.homeMixerFeatures .map { homeMixerFeatureOpt => homeMixerFeatureOpt.homeMixerFeaturesType.map { case hmf.HomeMixerFeaturesType.TwhinTweetEmbedding(homeMixerFeature) => homeMixerFeature case _ => throw new Exception("Unknown type returned") } } }.handle { case _ => Seq.fill(originalTweetIds.size)(None) } } private def getTwhinEmbeddingsFromReadableStore( originalTweetIds: Seq[Long] ): Future[Seq[Option[TwhinTweetEmbedding]]] = { val tweetIdVersionIdPairs = originalTweetIds.map(tweetId => (tweetId, versionId)) Future.collect(store.multiGet(tweetIdVersionIdPairs.toSet)).map { storeResponse => tweetIdVersionIdPairs.map { storeResponse.getOrElse(_, None) } } } private def getBatchedFeatureMap( candidatesBatch: Seq[CandidateWithFeatures[TweetCandidate]], callMiddleMan: Boolean ): Future[Seq[FeatureMap]] = { val originalTweetIds = candidatesBatch.map { candidate => { keyTotalCounter.incr() CandidatesUtil.getOriginalTweetId(candidate.candidate, candidate.features) } } val responseMap = if (callMiddleMan) getTwhinEmbeddingsFromHMF(originalTweetIds) else getTwhinEmbeddingsFromReadableStore(originalTweetIds) responseMap.map { response => response.map { twhinEmbeddingOpt => val floatTensor = { keyFoundCounter.incr() twhinEmbeddingOpt.map(twhinEmbedding => ml.FloatTensor(twhinEmbedding.embedding)) } val dataRecord = TwhinRebuildTweetEmbeddingsAdapter.adaptToDataRecords(floatTensor).asScala.head FeatureMap(TwhinRebuildTweetFeature, dataRecord) } } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val callMiddleMan = query.params(EnableHomeMixerFeaturesService) OffloadFuturePools.offloadBatchSeqToFutureSeq( candidates, getBatchedFeatureMap(_, callMiddleMan), batchSize) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinRebuildUserEngagementQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinRebuildUserEngagementEmbeddingsAdapter import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinRebuildUserEngagementFeatureRepository import com.twitter.ml.api.DataRecord import com.twitter.ml.api.{thriftscala => ml} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import com.twitter.util.Return import com.twitter.util.Throw import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object TwhinRebuildUserEngagementFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TwhinRebuildUserEngagementQueryFeatureHydrator @Inject() ( @Named(TwhinRebuildUserEngagementFeatureRepository) client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TwhinRebuildUserEngagement") override val features: Set[Feature[_, _]] = Set(TwhinRebuildUserEngagementFeature) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") private val keyTotalCounter = scopedStatsReceiver.counter("key/total") override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId Stitch.callFuture(client(Seq(userId))).map { results => keyTotalCounter.incr() val embedding: Option[ml.FloatTensor] = results(userId) match { case Return(value) => if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() else keyNotFoundCounter.incr() value case Throw(_) => keyFailureCounter.incr() None case _ => None } val dataRecord = TwhinRebuildUserEngagementEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head FeatureMapBuilder() .add(TwhinRebuildUserEngagementFeature, dataRecord) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinRebuildUserPositiveQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinRebuildUserPositiveEmbeddingsAdapter import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinRebuildUserPositiveEmbeddingsStore import com.twitter.ml.api.DataRecord import com.twitter.ml.api.{thriftscala => ml} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding import com.twitter.simclusters_v2.thriftscala.TwhinEmbeddingDataset import com.twitter.stitch.Stitch import com.twitter.storehaus.ReadableStore import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object TwhinRebuildUserPositiveFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TwhinRebuildUserPositiveQueryFeatureHydrator @Inject() ( @Named(TwhinRebuildUserPositiveEmbeddingsStore) store: ReadableStore[ (Long, Long), TwhinTweetEmbedding ], statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TwhinRebuildUserPositive") override val features: Set[Feature[_, _]] = Set(TwhinRebuildUserPositiveFeature) private val versionId = TwhinEmbeddingDataset.RefreshedTwhinTweet.value private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") private val keyTotalCounter = scopedStatsReceiver.counter("key/total") override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { Stitch .callFuture(store.get((query.getRequiredUserId, versionId))).map { resultOpt => keyTotalCounter.incr() resultOpt match { case Some(_) => keyFoundCounter.incr() case None => keyNotFoundCounter.incr() } val floatTensor = resultOpt.map(result => ml.FloatTensor(result.embedding)) val dataRecord = TwhinRebuildUserPositiveEmbeddingsAdapter .adaptToDataRecords(floatTensor).asScala.head FeatureMapBuilder().add(TwhinRebuildUserPositiveFeature, dataRecord).build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinTweetFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinTweetEmbeddingsAdapter import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinTweetEmbeddingsStore import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableHomeMixerFeaturesService import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer_features.{thriftscala => hmf} import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.util.Future import com.twitter.ml.api.{thriftscala => ml} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding import com.twitter.stitch.Stitch import com.twitter.storehaus.ReadableStore import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object TwhinTweetFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TwhinTweetFeatureHydrator @Inject() ( homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, @Named(TwhinTweetEmbeddingsStore) store: ReadableStore[Long, TwhinTweetEmbedding]) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TwhinTweet") override val features: Set[Feature[_, _]] = Set(TwhinTweetFeature) private val batchSize = 50 private def getTwhinEmbeddingsFromHMF( originalTweetIds: Seq[Long] ): Future[Seq[Option[TwhinTweetEmbedding]]] = { val keysSerialized = originalTweetIds.map(_.toString) val request = hmf.HomeMixerFeaturesRequest(keysSerialized, hmf.Cache.Twhin) val responseFut = homeMixerFeatureService.getHomeMixerFeatures(request) responseFut .map { response => response.homeMixerFeatures .map { homeMixerFeatureOpt => homeMixerFeatureOpt.homeMixerFeaturesType.map { case hmf.HomeMixerFeaturesType.TwhinTweetEmbedding(homeMixerFeature) => homeMixerFeature case _ => throw new Exception("Unknown type returned") } } }.handle { case _ => Seq.fill(originalTweetIds.size)(None) } } private def getTwhinEmbeddingsFromReadableStore( originalTweetIds: Seq[Long] ): Future[Seq[Option[TwhinTweetEmbedding]]] = { Future.collect(store.multiGet(originalTweetIds.toSet)).map { storeResponse => originalTweetIds.map { storeResponse.getOrElse(_, None) } } } private def getBatchedFeatureMap( candidatesBatch: Seq[CandidateWithFeatures[TweetCandidate]], callMiddleMan: Boolean ): Future[Seq[FeatureMap]] = { val originalTweetIds = candidatesBatch.map { candidate => CandidatesUtil.getOriginalTweetId(candidate.candidate, candidate.features) } val responseMap = if (callMiddleMan) getTwhinEmbeddingsFromHMF(originalTweetIds) else getTwhinEmbeddingsFromReadableStore(originalTweetIds) responseMap.map { response => response.map { twhinEmbeddingOpt => val floatTensor = twhinEmbeddingOpt.map(twhinEmbedding => ml.FloatTensor(twhinEmbedding.embedding)) val dataRecord = TwhinTweetEmbeddingsAdapter.adaptToDataRecords(floatTensor).asScala.head FeatureMap(TwhinTweetFeature, dataRecord) } } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val callMiddleMan = query.params(EnableHomeMixerFeaturesService) OffloadFuturePools.offloadBatchSeqToFutureSeq( candidates, getBatchedFeatureMap(_, callMiddleMan), batchSize) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinUserEngagementEmbeddingsAdapter import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserEngagementFeatureRepository import com.twitter.ml.api.DataRecord import com.twitter.ml.api.{thriftscala => ml} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import com.twitter.util.Return import com.twitter.util.Throw import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object TwhinUserEngagementFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TwhinUserEngagementQueryFeatureHydrator @Inject() ( @Named(TwhinUserEngagementFeatureRepository) client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TwhinUserEngagement") override val features: Set[Feature[_, _]] = Set(TwhinUserEngagementFeature) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId Stitch.callFuture(client(Seq(userId))).map { results => val embedding: Option[ml.FloatTensor] = results(userId) match { case Return(value) => if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() else keyNotFoundCounter.incr() value case Throw(_) => keyFailureCounter.incr() None case _ => None } val dataRecord = TwhinUserEngagementEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head FeatureMapBuilder() .add(TwhinUserEngagementFeature, dataRecord) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinUserFollowEmbeddingsAdapter import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserFollowFeatureRepository import com.twitter.ml.api.DataRecord import com.twitter.ml.api.{thriftscala => ml} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import com.twitter.util.Return import com.twitter.util.Throw import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object TwhinUserFollowFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TwhinUserFollowQueryFeatureHydrator @Inject() ( @Named(TwhinUserFollowFeatureRepository) client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TwhinUserFollow") override val features: Set[Feature[_, _]] = Set(TwhinUserFollowFeature) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId Stitch .callFuture(client(Seq(userId))) .map { results => val embedding: Option[ml.FloatTensor] = results(userId) match { case Return(value) => if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() else keyNotFoundCounter.incr() value case Throw(_) => keyFailureCounter.incr() None case _ => None } val dataRecord = TwhinUserFollowEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head FeatureMapBuilder() .add(TwhinUserFollowFeature, dataRecord) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserNegativeFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinUserNegativeEmbeddingsAdapter import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserNegativeEmbeddingsStore import com.twitter.ml.api.DataRecord import com.twitter.ml.api.{thriftscala => ml} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding import com.twitter.stitch.Stitch import com.twitter.storehaus.ReadableStore import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object TwhinUserNegativeFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TwhinUserNegativeQueryFeatureHydrator @Inject() ( @Named(TwhinUserNegativeEmbeddingsStore) store: ReadableStore[Long, TwhinTweetEmbedding]) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TwhinUserNegative") override val features: Set[Feature[_, _]] = Set(TwhinUserNegativeFeature) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { Stitch.callFuture(store.get(query.getRequiredUserId)).map { resultOpt => val floatTensor = resultOpt.map(result => ml.FloatTensor(result.embedding)) val dataRecord = TwhinUserNegativeEmbeddingsAdapter .adaptToDataRecords(floatTensor).asScala.head FeatureMapBuilder().add(TwhinUserNegativeFeature, dataRecord).build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinUserPositiveFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinUserPositiveEmbeddingsAdapter import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserPositiveEmbeddingsStore import com.twitter.ml.api.DataRecord import com.twitter.ml.api.{thriftscala => ml} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding import com.twitter.stitch.Stitch import com.twitter.storehaus.ReadableStore import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object TwhinUserPositiveFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TwhinUserPositiveQueryFeatureHydrator @Inject() ( @Named(TwhinUserPositiveEmbeddingsStore) store: ReadableStore[Long, TwhinTweetEmbedding]) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TwhinUserPositive") override val features: Set[Feature[_, _]] = Set(TwhinUserPositiveFeature) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { Stitch.callFuture(store.get(query.getRequiredUserId)).map { resultOpt => val floatTensor = resultOpt.map(result => ml.FloatTensor(result.embedding)) val dataRecord = TwhinUserPositiveEmbeddingsAdapter .adaptToDataRecords(floatTensor).asScala.head FeatureMapBuilder().add(TwhinUserPositiveFeature, dataRecord).build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TwhinVideoFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinVideoEmbeddingsAdapter import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinVideoEmbeddingsStore import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.ml.api.DataRecord import com.twitter.ml.api.{thriftscala => ml} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding import com.twitter.stitch.Stitch import com.twitter.storehaus.ReadableStore import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object TwhinVideoFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TwhinVideoFeatureHydrator @Inject() ( @Named(TwhinVideoEmbeddingsStore) store: ReadableStore[Long, TwhinTweetEmbedding]) extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TwhinVideo") override val features: Set[Feature[_, _]] = Set(TwhinVideoFeature) override def apply( query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap ): Stitch[FeatureMap] = OffloadFuturePools.offloadFuture { val originalTweetId = CandidatesUtil.getOriginalTweetId(candidate, existingFeatures) store.get(originalTweetId).map { resultOpt => val floatTensor = resultOpt.map(result => ml.FloatTensor(result.embedding)) val dataRecord = TwhinVideoEmbeddingsAdapter.adaptToDataRecords(floatTensor).asScala.head FeatureMapBuilder().add(TwhinVideoFeature, dataRecord).build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UnifiedUserActionsUserIdentifierFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.unified_counter.service.UuaUserIdentifierClientColumn import javax.inject.Inject import javax.inject.Singleton @Singleton class UnifiedUserActionsUserIdentifierFeatureHydrator @Inject() ( uuaUserIdentifierClientColumn: UuaUserIdentifierClientColumn) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UnifiedUserActionsUserIdentifier") override val features: Set[Feature[_, _]] = Set( UuaUserGenderFeature, UuaUserStateFeature, UuaUserAgeBucketFeature ) private val DefaultFeatureMap = FeatureMapBuilder() .add(UuaUserGenderFeature, None) .add(UuaUserStateFeature, None) .add(UuaUserAgeBucketFeature, None) .build() override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { uuaUserIdentifierClientColumn.fetcher .fetch(query.getRequiredUserId.toString) .map { response => response.v match { case Some(userInfo) => val gender = userInfo.userGender.map(_.toString) val state = userInfo.userState.map(_.value.toLong) val ageBucket = userInfo.userAgeBucket.map(_.toString) FeatureMapBuilder() .add(UuaUserGenderFeature, gender) .add(UuaUserStateFeature, state) .add(UuaUserAgeBucketFeature, ageBucket) .build() case _ => DefaultFeatureMap } } .rescue { case _: Throwable => Stitch.value(DefaultFeatureMap) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserActionByteArrayQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.UserActionsByteArrayFeature import com.twitter.home_mixer.param.HomeGlobalParams.UserActionsMaxCount import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.user_history_transformer.user_actions.UserActionSequenceMhClientColumn import com.twitter.user_history_transformer.domain.AggregationAlgorithmV1 import com.twitter.user_history_transformer.domain.AggregationConfig import com.twitter.user_history_transformer.domain.AggregationProcessor import com.twitter.user_history_transformer.domain.UserActionSequenceUtils import com.twitter.user_history_transformer.util.SchemaUtils import com.x.user_action_sequence.thriftscala.UserActionSequenceDataContainer.OrderedAggregatedUserActionList import com.x.user_action_sequence.{thriftscala => t} import javax.inject.Inject import javax.inject.Singleton @Singleton class UserActionsArrayByteQueryFeatureHydrator @Inject() ( userActionSequenceMhClientColumn: UserActionSequenceMhClientColumn, statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserActionsByteArray") override val features: Set[Feature[_, _]] = Set(UserActionsByteArrayFeature) private val DefaultFeatureMap = FeatureMap(UserActionsByteArrayFeature, None) private val windowTimeMs = 5 * 60 * 1000 private val aggregationProcessor = new AggregationProcessor( AggregationConfig( postProcessorSeq = Seq.empty, windowTimeMs = windowTimeMs, maxLength = 1024, aggregationAlgorithm = AggregationAlgorithmV1 ) ) private def hasNegativeValue(value: Option[Long]): Boolean = value.exists(_ < 0) private def hasNegativeValues(aggregatedUserAction: t.AggregatedUserAction): Boolean = { if (hasNegativeValue(aggregatedUserAction.userId)) return true aggregatedUserAction.tweetInfo.exists { tweetInfo => val fieldsToCheck = List( tweetInfo.tweetId, tweetInfo.authorId, tweetInfo.retweetingTweetId, tweetInfo.quotingTweetId, tweetInfo.replyingTweetId, tweetInfo.quotedTweetId, tweetInfo.inReplyToTweetId, tweetInfo.retweetedTweetId, tweetInfo.editedTweetId ) fieldsToCheck.exists(hasNegativeValue) } } override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = OffloadFuturePools.offloadStitch { userActionSequenceMhClientColumn.fetcher.fetch(query.getRequiredUserId).map { response => val featureMap = response.v.map { userActionsSeq => val decompressedUserActionSeq = UserActionSequenceUtils.expand(userActionsSeq, statsReceiver) val downsampledUserActionSeq = UserActionSequenceUtils.mhToKafka(decompressedUserActionSeq) val aggregatedUserActions = aggregationProcessor .process(downsampledUserActionSeq) .filterNot(hasNegativeValues) .takeRight(query.params(UserActionsMaxCount)) val filteredUserActionSeq = userActionsSeq.copy( userActionsData = Some( OrderedAggregatedUserActionList( t.AggregatedUserActionList(aggregatedUserActions = Some(aggregatedUserActions)) ) ) ) val actions = SchemaUtils.convertUserActionSequenceThriftToProtobuf(filteredUserActionSeq) FeatureMap(UserActionsByteArrayFeature, Some(actions.toByteArray)) } featureMap.getOrElse(DefaultFeatureMap) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserActionsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.UserActionsContainsExplicitSignalsFeature import com.twitter.home_mixer.model.HomeFeatures.UserActionsFeature import com.twitter.home_mixer.model.HomeFeatures.UserActionsSizeFeature import com.twitter.home_mixer.param.HomeGlobalParams.UserActionsMaxCount import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableDenseUserActionsHydrationParam import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.user_history_transformer.user_actions.UserActionSequenceMhClientColumn import com.twitter.user_history_transformer.domain.AggregationAlgorithmV1 import com.twitter.user_history_transformer.domain.AggregationAlgorithmWithoutHomeFilter import com.twitter.user_history_transformer.domain.AggregationConfig import com.twitter.user_history_transformer.domain.AggregationProcessor import com.twitter.user_history_transformer.domain.UserActionSequenceUtils import com.twitter.user_history_transformer.util.SchemaUtils import com.x.user_action_sequence.thriftscala.ActionName.ClientTweetRecapDwelled import com.x.user_action_sequence.thriftscala.ActionName.ClientTweetRecapNotDwelled import com.x.user_action_sequence.thriftscala.ActionName import com.x.user_action_sequence.thriftscala.AggregatedUserAction import com.x.user_action_sequence.thriftscala.UserActionSequenceDataContainer.OrderedAggregatedUserActionList import com.x.user_action_sequence.{thriftscala => t} import javax.inject.Inject import javax.inject.Singleton @Singleton class UserActionsQueryFeatureHydrator @Inject() ( userActionSequenceMhClientColumn: UserActionSequenceMhClientColumn, statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserActions") override val features: Set[Feature[_, _]] = Set( UserActionsFeature, UserActionsSizeFeature, UserActionsContainsExplicitSignalsFeature ) private val userAggregatedActionSeqlengthStat = statsReceiver.stat("UserActionsQueryFeatureHydrator", "length") private val DefaultFeatureMap = FeatureMapBuilder() .add(UserActionsFeature, None) .add(UserActionsSizeFeature, None) .add(UserActionsContainsExplicitSignalsFeature, false) .build() private val windowTimeMs = 5 * 60 * 1000 private val ExcludedDwellActions: Set[ActionName] = Set(ClientTweetRecapDwelled, ClientTweetRecapNotDwelled) private def filterDwells( aggAction: AggregatedUserAction ): Boolean = { aggAction.actions .getOrElse(Seq.empty) .exists { _.actionName.exists { name => !ExcludedDwellActions.contains(name) } } } private val aggregationProcessor = new AggregationProcessor( AggregationConfig( postProcessorSeq = Seq.empty, windowTimeMs = windowTimeMs, maxLength = 1024, aggregationAlgorithm = AggregationAlgorithmV1, ) ) private val denseAggregationProcessor = new AggregationProcessor( AggregationConfig( postProcessorSeq = Seq.empty, windowTimeMs = windowTimeMs, maxLength = 1024, aggregationAlgorithm = AggregationAlgorithmWithoutHomeFilter, aggActionFilterFuncGenerator = (_, _) => { aggAction => filterDwells(aggAction) } ) ) private def hasNegativeValue(value: Option[Long]): Boolean = value.exists(_ < 0) private def hasNegativeValues(aggregatedUserAction: t.AggregatedUserAction): Boolean = { if (hasNegativeValue(aggregatedUserAction.userId)) return true aggregatedUserAction.tweetInfo.exists { tweetInfo => val fieldsToCheck = List( tweetInfo.tweetId, tweetInfo.authorId, tweetInfo.retweetingTweetId, tweetInfo.quotingTweetId, tweetInfo.replyingTweetId, tweetInfo.quotedTweetId, tweetInfo.inReplyToTweetId, tweetInfo.retweetedTweetId, tweetInfo.editedTweetId ) fieldsToCheck.exists(hasNegativeValue) } } override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val processor = if (query.params(EnableDenseUserActionsHydrationParam)) denseAggregationProcessor else aggregationProcessor OffloadFuturePools.offloadStitch { userActionSequenceMhClientColumn.fetcher.fetch(query.getRequiredUserId).map { response => val featureMap = response.v.map { userActionsSeq => val decompressedUserActionSeq = UserActionSequenceUtils.expand(userActionsSeq, statsReceiver) val aggregatedUserActions = processor .process(decompressedUserActionSeq) .filterNot(hasNegativeValues) .takeRight(query.params(UserActionsMaxCount)) val size = aggregatedUserActions.length userAggregatedActionSeqlengthStat.add(size) val hasExplicitSignals = UserActionSequenceUtils.hasExplicitSignals(decompressedUserActionSeq) val filteredUserActionSeq = userActionsSeq.copy( userActionsData = Some( OrderedAggregatedUserActionList( t.AggregatedUserActionList(aggregatedUserActions = Some(aggregatedUserActions)) ) ) ) val actions = SchemaUtils.convertUserActionSequenceThriftToProtobuf(filteredUserActionSeq) FeatureMapBuilder() .add(UserActionsFeature, Some(actions)) .add(UserActionsSizeFeature, Some(size)) .add(UserActionsContainsExplicitSignalsFeature, hasExplicitSignals) .build() } featureMap.getOrElse(DefaultFeatureMap) } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserEngagedGrokCategoriesFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.trends.trip.EngagedGrokTopicsAndTagsMonthlyOnUserClientColumn import javax.inject.Inject import javax.inject.Singleton object UserSubLevelCategoriesFeature extends Feature[TweetCandidate, Seq[(Long, Double)]] @Singleton class UserEngagedGrokCategoriesFeatureHydrator @Inject() ( engagedGrokTopicsAndTagMonthlysOnUserClientColumn: EngagedGrokTopicsAndTagsMonthlyOnUserClientColumn) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserEngagedGrokCategories") override val features: Set[Feature[_, _]] = Set(UserSubLevelCategoriesFeature) private def fetchSubLevelEntities(userId: Long): Stitch[Seq[(Long, Double)]] = { engagedGrokTopicsAndTagMonthlysOnUserClientColumn.fetcher.fetch(userId).map { result => result.v .flatMap(_.subLevelEntities).getOrElse(Seq.empty) .take(3) .map(entityInfo => (entityInfo.entityId, entityInfo.score)) } } override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId fetchSubLevelEntities(userId).map { subEntities => FeatureMapBuilder() .add(UserSubLevelCategoriesFeature, subEntities) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserEngagedLanguagesFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.UserEngagedLanguagesFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.language.user.NormalizedEngagedTweetLanguagesOnUserClientColumn import com.twitter.language.types.{thriftscala => lg} import javax.inject.Inject import javax.inject.Singleton @Singleton class UserEngagedLanguagesFeatureHydrator @Inject() ( engagedLanguageOnUserColumn: NormalizedEngagedTweetLanguagesOnUserClientColumn) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserEngagedLanguages") override def features: Set[Feature[_, _]] = Set(UserEngagedLanguagesFeature) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { engagedLanguageOnUserColumn.fetcher .fetch(query.getRequiredUserId, Some(lg.LanguageType.User)).map { result => FeatureMapBuilder() .add(UserEngagedLanguagesFeature, result.v.getOrElse(Seq.empty).toSet) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserEngagementGrokTagFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.content_understanding.UserTopTagsMhClientColumn import com.twitter.strato.generated.client.trends.trip.UserAssociatedTopicsClientColumn import com.twitter.trends.trip_v1.user_topics.{thriftscala => ut} import javax.inject.Inject import javax.inject.Singleton object RealtimeUserEngagedGrokTagFeature extends Feature[TweetCandidate, Seq[(String, Option[Double])]] object EvergreenUserEngagedGrokTagFeature extends Feature[TweetCandidate, Seq[(String, Option[Double])]] @Singleton class UserEngagementGrokTagFeatureHydrator @Inject() ( evergreenUserTopTags: UserTopTagsMhClientColumn, tripUserTopTags: UserAssociatedTopicsClientColumn) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "UserEngagementGrokTag") override val features: Set[Feature[_, _]] = Set(RealtimeUserEngagedGrokTagFeature, EvergreenUserEngagedGrokTagFeature) def fetchRealTimeUserTags(userId: Long): Stitch[Seq[(String, Option[Double])]] = { tripUserTopTags.fetcher.fetch(ut.UserTopicDomain(userId = userId, sourceId = None)).map { result => result.v.flatMap(_.tags).getOrElse(Seq.empty[ut.TagCandidate]).map(tc => (tc.tag, tc.score)) } } def fetchEvergreenUserTags(userId: Long): Stitch[Seq[(String, Option[Double])]] = { evergreenUserTopTags.fetcher.fetch(userId).map { view => view.v.map(_.tags.map(tc => (tc.tag, Some(tc.score)))).getOrElse(Seq.empty) } } override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId Stitch.join(fetchRealTimeUserTags(userId), fetchEvergreenUserTags(userId)).map { case (realtime, evergreen) => FeatureMapBuilder() .add(RealtimeUserEngagedGrokTagFeature, realtime) .add(EvergreenUserEngagedGrokTagFeature, evergreen) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserFrequentLocationHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.geoduck.common.thriftscala.TransactionLocation import com.twitter.geoduck.common.{thriftscala => t} import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.product_mixer.component_library.feature.location.{ Location => ProductMixerLocation } import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.cache.ExpiringLruInProcessCache import com.twitter.servo.cache.InProcessCache import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.geo.service.UserLocationClientColumn import javax.inject.Inject import javax.inject.Singleton object UserFrequentLocationHydrator { private val BaseTTLMinutes = 60 * 24 private val TTL = (BaseTTLMinutes + scala.util.Random.nextInt(60)).minutes val cache: InProcessCache[Long, Option[TransactionLocation]] = new ExpiringLruInProcessCache[Long, Option[TransactionLocation]]( ttl = TTL, maximumSize = 150 * 1000 // Cache up to 150k users ) } @Singleton class UserFrequentLocationHydrator @Inject() ( userLocationClientColumn: UserLocationClientColumn) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "UserFrequentLocationHydrator") override val features: Set[Feature[_, _]] = Set(LocationFeature) private val PlaceQuery = t.PlaceQuery( placeTypes = Some( Set( t.PlaceType.Neighborhood, t.PlaceType.City, t.PlaceType.Metro, t.PlaceType.Admin1, t.PlaceType.Country ) ) ) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)).distinct val (hitIds, missIds) = authorIds.partition(id => UserFrequentLocationHydrator.cache.get(id).isDefined) val cachedOptionMap: Map[Long, Option[TransactionLocation]] = hitIds.map(id => id -> UserFrequentLocationHydrator.cache.get(id).get).toMap val fetchCacheMisses: Stitch[Map[Long, TransactionLocation]] = if (missIds.isEmpty) { Stitch.value(Map.empty[Long, TransactionLocation]) } else { userLocationClientColumn.fetcher .fetch( key = Unit, t.UserLocationRequest( userIds = missIds, placeQuery = Some(PlaceQuery) ) ) .map { response => response.v.toList.flatMap(_._1).toMap } .handle { case e => Map.empty[Long, TransactionLocation] } } val allLocationsStitch: Stitch[Map[Long, TransactionLocation]] = fetchCacheMisses.map { fetchedMap => missIds.foreach { id => val locOpt: Option[TransactionLocation] = fetchedMap.get(id) UserFrequentLocationHydrator.cache.set(id, locOpt) } val cachedLocs: Map[Long, TransactionLocation] = cachedOptionMap.collect { case (id, Some(loc)) => id -> loc } cachedLocs ++ fetchedMap } allLocationsStitch.map { allLocations => candidates.map { candidate => val locOpt = for { authorId <- candidate.features.getOrElse(AuthorIdFeature, None) loc <- allLocations.get(authorId) } yield loc locOpt .map { transactionLocation => val placeMap = transactionLocation.placeMap val locationDetails = ProductMixerLocation( neighborhood = placeMap .flatMap(_.get(t.PlaceType.Neighborhood)) .flatMap(_.headOption), city = placeMap .flatMap(_.get(t.PlaceType.City)) .flatMap(_.headOption), metro = placeMap .flatMap(_.get(t.PlaceType.Metro)) .flatMap(_.headOption), region = placeMap .flatMap(_.get(t.PlaceType.Admin1)) .flatMap(_.headOption), country = placeMap .flatMap(_.get(t.PlaceType.Country)) .flatMap(_.headOption) ) FeatureMapBuilder() .add(LocationFeature, Option(locationDetails)) .build() } .getOrElse { FeatureMapBuilder() .add(LocationFeature, None) .build() } } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserFrequentLocationQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.product_mixer.component_library.feature.location.{Location => ProductLocation} import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.geoduck.common.{thriftscala => t} import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.geo.service.UserLocationClientColumn import javax.inject.Inject import javax.inject.Singleton @Singleton class UserFrequentLocationQueryFeatureHydrator @Inject() ( userLocationClientColumn: UserLocationClientColumn) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserFrequentLocationQuery") override val features: Set[Feature[_, _]] = Set(LocationFeature) private val PlaceQuery = t.PlaceQuery( placeTypes = Some( Set( t.PlaceType.Neighborhood, t.PlaceType.City, t.PlaceType.Metro, t.PlaceType.Admin1, t.PlaceType.Country ) ) ) private val DefaultFeatureMap = FeatureMapBuilder() .add(LocationFeature, None) .build() override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId val locationStitch = userLocationClientColumn.fetcher .fetch( key = Unit, t.UserLocationRequest( userIds = Seq(userId), placeQuery = Some(PlaceQuery) ) ) .map { response => response.v.toList.flatMap(_._1).toMap.get(userId) } locationStitch .map { locationOpt => locationOpt .map { location => val placeMap = location.placeMap val locationDetails = ProductLocation( neighborhood = placeMap .flatMap(_.get(t.PlaceType.Neighborhood)) .flatMap(_.headOption), city = placeMap .flatMap(_.get(t.PlaceType.City)) .flatMap(_.headOption), metro = placeMap .flatMap(_.get(t.PlaceType.Metro)) .flatMap(_.headOption), region = placeMap .flatMap(_.get(t.PlaceType.Admin1)) .flatMap(_.headOption), country = placeMap .flatMap(_.get(t.PlaceType.Country)) .flatMap(_.headOption) ) FeatureMapBuilder() .add(LocationFeature, Option(locationDetails)) .build() } .getOrElse { DefaultFeatureMap } }.handle { case e => DefaultFeatureMap } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserHistoryTransformerEmbeddingQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.TransformerByteEmbeddingsAdapter import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.TransformerEmbeddingsAdapter import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.UserHistoryTransformerEmbeddingsHomeBlueAdapter import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.UserHistoryTransformerEmbeddingsHomeGreenAdapter import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.UserHistoryTransformerEmbeddingsJointBlueAdapter import com.twitter.home_mixer_features.thriftscala.HomeMixerFeaturesType import com.twitter.home_mixer_features.{thriftscala => hmf} import com.twitter.ml.api.DataRecord import com.twitter.ml.api.{thriftscala => ml} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.util.Future import java.nio.ByteBuffer import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ object UserHistoryTransformerEmbeddingHomeBlueFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object UserHistoryTransformerEmbeddingHomeGreenFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object UserHistoryTransformerEmbeddingJointBlueFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class UserHistoryTransformerEmbeddingQueryFeatureHydratorBuilder @Inject() ( homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, statsReceiver: StatsReceiver) { def buildHomeBlueHydrator(): UserHistoryTransformerFloatEmbeddingQueryFeatureHydrator = { UserHistoryTransformerFloatEmbeddingQueryFeatureHydrator( FeatureHydratorIdentifier("HomeBlueTransformerEmbeddingQueryFeatureHydrator"), homeMixerFeatureService, statsReceiver, UserHistoryTransformerEmbeddingHomeBlueFeature, hmf.Cache.TransformerUserEmbeddings, UserHistoryTransformerEmbeddingsHomeBlueAdapter ) } def buildHomeGreenHydrator(): UserHistoryTransformerByteEmbeddingQueryFeatureHydrator = { UserHistoryTransformerByteEmbeddingQueryFeatureHydrator( FeatureHydratorIdentifier("HomeGreenTransformerEmbeddingQueryFeatureHydrator"), homeMixerFeatureService, statsReceiver, UserHistoryTransformerEmbeddingHomeGreenFeature, hmf.Cache.TransformerUserEmbeddingsGreen, UserHistoryTransformerEmbeddingsHomeGreenAdapter ) } def buildJointBlueHydrator(): UserHistoryTransformerByteEmbeddingQueryFeatureHydrator = { UserHistoryTransformerByteEmbeddingQueryFeatureHydrator( FeatureHydratorIdentifier("JointBlueTransformerEmbeddingQueryFeatureHydrator"), homeMixerFeatureService, statsReceiver, UserHistoryTransformerEmbeddingJointBlueFeature, hmf.Cache.TransformerUserJointEmbeddingsBlue, UserHistoryTransformerEmbeddingsJointBlueAdapter ) } } case class UserHistoryTransformerFloatEmbeddingQueryFeatureHydrator( override val identifier: FeatureHydratorIdentifier, override val homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, override val statsReceiver: StatsReceiver, override val embeddingFeature: DataRecordInAFeature[PipelineQuery], override val cacheType: hmf.Cache, transformerEmbeddingsAdapter: TransformerEmbeddingsAdapter) extends BaseUserHistoryTransformerEmbeddingQueryFeatureHydrator { override def responseToDataRecord(homeMixerFeaturesType: HomeMixerFeaturesType): DataRecord = { val embedding = homeMixerFeaturesType match { case hmf.HomeMixerFeaturesType.RawEmbedding(floatEmbedding) => floatEmbedding case other => wrongTypeCounter.incr() throw new Exception( f"Type not matching. Expected RawEmbedding but got ${other.getClass.getSimpleName}") } val tensor = ml.FloatTensor(embedding) transformerEmbeddingsAdapter.adaptToDataRecords(Some(tensor)).asScala.head } } case class UserHistoryTransformerByteEmbeddingQueryFeatureHydrator( override val identifier: FeatureHydratorIdentifier, override val homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint, override val statsReceiver: StatsReceiver, override val embeddingFeature: DataRecordInAFeature[PipelineQuery], override val cacheType: hmf.Cache, transformerByteEmbeddingsAdapter: TransformerByteEmbeddingsAdapter) extends BaseUserHistoryTransformerEmbeddingQueryFeatureHydrator { override def responseToDataRecord(homeMixerFeaturesType: HomeMixerFeaturesType): DataRecord = { val embedding = homeMixerFeaturesType match { case hmf.HomeMixerFeaturesType.RawByteEmbedding(byteEmbedding) => byteEmbedding case other => wrongTypeCounter.incr() throw new Exception( f"Type not matching. Expected RawByteEmbedding but got ${other.getClass.getSimpleName}") } val tensor = ml.RawTypedTensor(ml.DataType.Byte, convertToByteBuffer(embedding)) transformerByteEmbeddingsAdapter.adaptToDataRecords(Some(tensor)).asScala.head } private def convertToByteBuffer(byteArray: Seq[Byte]): ByteBuffer = { val buffer = ByteBuffer.allocate(byteArray.size) byteArray.foreach(buffer.put) buffer.flip buffer } } trait BaseUserHistoryTransformerEmbeddingQueryFeatureHydrator extends QueryFeatureHydrator[PipelineQuery] { def homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint def statsReceiver: StatsReceiver def embeddingFeature: DataRecordInAFeature[PipelineQuery] def cacheType: hmf.Cache override val features: Set[Feature[_, _]] = Set(embeddingFeature) protected val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) protected val keyFoundCounter = scopedStatsReceiver.counter("key/found") protected val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") protected val wrongTypeCounter = scopedStatsReceiver.counter("key/wrongType") def responseToDataRecord(homeMixerFeaturesType: HomeMixerFeaturesType): DataRecord private def getFeatureMap( pipelineQuery: PipelineQuery ): Future[FeatureMap] = { val keysSerialized = Seq(pipelineQuery.getRequiredUserId.toString) val request = hmf.HomeMixerFeaturesRequest(keysSerialized, cacheType) val responseFut = homeMixerFeatureService.getHomeMixerFeatures(request) responseFut .map { response => response.homeMixerFeatures.headOption.flatMap { homeMixerFeatureOpt => homeMixerFeatureOpt.homeMixerFeaturesType match { case Some(homeMixerFeaturesType) => keyFoundCounter.incr() Some(responseToDataRecord(homeMixerFeaturesType)) case None => keyNotFoundCounter.incr() None } } }.handle { case _ => None } .map { case Some(dataRecord) => FeatureMap(embeddingFeature, dataRecord) case None => FeatureMap(embeddingFeature, new DataRecord()) } } override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { Stitch.callFuture(getFeatureMap(query)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserLanguagesFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserLanguagesRepository import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.search.common.constants.{thriftscala => scc} import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton object UserLanguagesFeature extends Feature[PipelineQuery, Seq[scc.ThriftLanguage]] @Singleton case class UserLanguagesFeatureHydrator @Inject() ( @Named(UserLanguagesRepository) client: KeyValueRepository[Seq[Long], Long, Seq[ scc.ThriftLanguage ]], statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserLanguages") override val features: Set[Feature[_, _]] = Set(UserLanguagesFeature) override val statScope: String = identifier.toString override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val key = query.getRequiredUserId Stitch.callFuture(client(Seq(key))).map { result => val feature = observedGet(key = Some(key), keyValueResult = result).map(_.getOrElse(Seq.empty)) FeatureMapBuilder() .add(UserLanguagesFeature, feature) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserLargeEmbeddingsFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.UserLargeEmbeddingsFeature import com.twitter.home_mixer.model.HomeLargeEmbeddingsFeatures.UserLargeEmbeddingsKeyFeature import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableLargeEmbeddingsFeatureHydrationParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelNameParam import com.twitter.home_mixer_features.{thriftscala => hmf} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.large_embeddings.HashingFeatureParams import com.twitter.timelines.prediction.adapters.large_embeddings.HomeMixerLargeEmbeddingsFeatureHydrator import com.twitter.timelines.prediction.adapters.large_embeddings.LargeEmbeddingsAdapter import com.twitter.timelines.prediction.adapters.large_embeddings.UserLargeEmbeddingsAdapter import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton @Singleton class UserLargeEmbeddingsFeatureHydrator @Inject() ( statsReceiver: StatsReceiver, override val homeMixerFeatureService: hmf.HomeMixerFeatures.MethodPerEndpoint) extends QueryFeatureHydrator[PipelineQuery] with Conditionally[PipelineQuery] with HomeMixerLargeEmbeddingsFeatureHydrator { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserLargeEmbeddings") override val features: Set[Feature[_, _]] = Set(UserLargeEmbeddingsFeature, UserLargeEmbeddingsKeyFeature) override val adapter: LargeEmbeddingsAdapter = UserLargeEmbeddingsAdapter override val cacheType: hmf.Cache = hmf.Cache.UserLargeEmbeddings override val scopedStatsReceiver: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableLargeEmbeddingsFeatureHydrationParam) // Hashing Features override val defaultHashingFeatureParams: HashingFeatureParams = HashingFeatureParams( scales = Seq(1681734645L, 1546314972L), biases = Seq(2701200313L, 1873259806L), modulus = 3055559939L, bucketSize = 10000000L, ) override val modelName2HashingFeatureParams: Map[String, HashingFeatureParams] = Map( "hr_video_prod__v3_realtime" -> HashingFeatureParams( scales = Seq(1341131000L, 519927459L), biases = Seq(2924993425L, 294422133L), modulus = 3109207999L, bucketSize = 10000000L, ), "hr_video_prod__v2_lembeds" -> HashingFeatureParams( scales = Seq(214226227L, 561611689L), biases = Seq(182790211L, 330327483L), modulus = 816016163L, bucketSize = 1000000L, ), "hr_prod__v4_embeds_230M" -> HashingFeatureParams( scales = Seq(196742702L, 1852108266L), biases = Seq(1935840681L, 167407236L), modulus = 2859568897L, bucketSize = 100000000L, ), "hr_prod__v5_embeds_230M_and_transformer" -> HashingFeatureParams( scales = Seq(196742702L, 1852108266L), biases = Seq(1935840681L, 167407236L), modulus = 2859568897L, bucketSize = 100000000L, ), "hr_prod__v5_watchtime" -> HashingFeatureParams( scales = Seq(45230244L, 676046872L), biases = Seq(866394657L, 1019127517L), modulus = 1047809363L, bucketSize = 100000000L, ), "hr_prod__v6_transformer_v2" -> HashingFeatureParams( scales = Seq(196742702L, 1852108266L), biases = Seq(1935840681L, 167407236L), modulus = 2859568897L, bucketSize = 100000000L, ), "hr_prod__v6_mixed_training" -> HashingFeatureParams( scales = Seq(196742702L, 1852108266L), biases = Seq(1935840681L, 167407236L), modulus = 2859568897L, bucketSize = 100000000L, ), "hr_prod__v6_transformer_v2_kafka_merge_join" -> HashingFeatureParams( scales = Seq(196742702L, 1852108266L), biases = Seq(1935840681L, 167407236L), modulus = 2859568897L, bucketSize = 100000000L, ), "hr_prod__v6_transformer_v2_realtime_debias_21apr" -> HashingFeatureParams( scales = Seq(196742702L, 1852108266L), biases = Seq(1935840681L, 167407236L), modulus = 2859568897L, bucketSize = 100000000L, ), "hr_video_prod__v4_realtime" -> HashingFeatureParams( scales = Seq(1341131000L, 519927459L), biases = Seq(2924993425L, 294422133L), modulus = 3109207999L, bucketSize = 10000000L, ), "hr_video_prod__v4_realtime_mergehead" -> HashingFeatureParams( scales = Seq(1341131000L, 519927459L), biases = Seq(2924993425L, 294422133L), modulus = 3109207999L, bucketSize = 10000000L, ), ) private def getFeatureMap( pipelineQuery: PipelineQuery ): Future[FeatureMap] = { val userId = pipelineQuery.getRequiredUserId val modelName = pipelineQuery.params(ModelNameParam) val responseMap = getLargeEmbeddings(userId, modelName) responseMap.map { response => FeatureMapBuilder() .add(UserLargeEmbeddingsFeature, response.dataRecord) .add(UserLargeEmbeddingsKeyFeature, response.hashedKeys) .build() } } override def hydrate( query: PipelineQuery ): Stitch[FeatureMap] = Stitch.callFuture(getFeatureMap(query)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserStateQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.user_health.v1.{thriftscala => uhv1} import com.twitter.timelines.user_health.{thriftscala => uh} import com.twitter.user_session_store.ReadOnlyUserSessionStore import com.twitter.user_session_store.ReadRequest import com.twitter.user_session_store.UserSessionDataset import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset import javax.inject.Inject import javax.inject.Singleton @Singleton case class UserStateQueryFeatureHydrator @Inject() ( userSessionStore: ReadOnlyUserSessionStore) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserState") override val features: Set[Feature[_, _]] = Set(UserStateFeature) private val datasets: Set[UserSessionDataset] = Set(UserSessionDataset.UserHealth) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { userSessionStore .read(ReadRequest(query.getRequiredUserId, datasets)) .map { userSession => val userState = userSession.flatMap { _.userHealth match { case Some(uh.UserHealth.V1(uhv1.UserHealth(userState))) => userState case _ => None } } FeatureMapBuilder() .add(UserStateFeature, userState) .build() } } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserSubscriptionQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.subscription_services.subscription_verification.HasNoAdsBenefitOnUserClientColumn import javax.inject.Inject import javax.inject.Singleton object NoAdsTierFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Boolean] { override val defaultValue: Boolean = false } @Singleton case class UserSubscriptionQueryFeatureHydrator @Inject() ( hasNoAdsBenefitOnUserClientColumn: HasNoAdsBenefitOnUserClientColumn) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserSubscription") override val features: Set[Feature[_, _]] = Set(NoAdsTierFeature) override def hydrate( query: PipelineQuery ): Stitch[FeatureMap] = hasNoAdsBenefitOnUserClientColumn.fetcher .fetch(query.getRequiredUserId) .map { result => FeatureMapBuilder() .add(NoAdsTierFeature, result.v.getOrElse(false)) .build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UserUnderstandableLangaugesFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.UserUnderstandableLanguagesFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableGrokAutoTranslateLanguageFilter import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.strato.generated.client.language.user.GrokAutoTranslateUnderstandableLanguagesOnUserClientColumn import javax.inject.Inject import javax.inject.Singleton @Singleton class UserUnderstandableLanguagesFeatureHydrator @Inject() ( understandableLanguagesClientColumn: GrokAutoTranslateUnderstandableLanguagesOnUserClientColumn, override val statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] with Conditionally[PipelineQuery] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserUnderstandableLanguages") override val features: Set[Feature[_, _]] = Set(UserUnderstandableLanguagesFeature) override val statScope: String = identifier.toString private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val successCounter = scopedStatsReceiver.counter("success") private val failedCounter = scopedStatsReceiver.counter("failure") private val DefaultFeatureMap = FeatureMap(UserUnderstandableLanguagesFeature, Seq.empty) private val fetcher: Fetcher[Long, Unit, Seq[String]] = understandableLanguagesClientColumn.fetcher override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableGrokAutoTranslateLanguageFilter) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val key = query.getRequiredUserId fetcher .fetch(key, ()).map { result => successCounter.incr() FeatureMap(UserUnderstandableLanguagesFeature, result.v.getOrElse(Seq.empty)) }.rescue { case _ => failedCounter.incr() Stitch.value(DefaultFeatureMap) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/UtegFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.FavoritedByCountFeature import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.home_mixer.model.HomeFeatures.RepliedByCountFeature import com.twitter.home_mixer.model.HomeFeatures.RepliedByEngagerIdsFeature import com.twitter.home_mixer.model.HomeFeatures.RetweetedByCountFeature import com.twitter.home_mixer.model.HomeFeatures.RetweetedByEngagerIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.UtegSocialProofRepository import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.recos.recos_common.{thriftscala => rc} import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class UtegFeatureHydrator @Inject() ( @Named(UtegSocialProofRepository) client: KeyValueRepository[ (Seq[Long], (Long, Map[Long, Double])), Long, uteg.TweetRecommendation ]) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Uteg") override val features: Set[Feature[_, _]] = Set( FavoritedByUserIdsFeature, RetweetedByEngagerIdsFeature, RepliedByEngagerIdsFeature, FavoritedByCountFeature, RetweetedByCountFeature, RepliedByCountFeature ) override def onlyIf(query: PipelineQuery): Boolean = query.features .exists(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double]).nonEmpty) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val seedUserWeights = query.features.map(_.get(RealGraphInNetworkScoresFeature)).get val sourceTweetIds = candidates.flatMap(_.features.getOrElse(SourceTweetIdFeature, None)) val inReplyToTweetIds = candidates.flatMap(_.features.getOrElse(InReplyToTweetIdFeature, None)) val tweetIds = candidates.map(_.candidate.id) val tweetIdsToSend = (tweetIds ++ sourceTweetIds ++ inReplyToTweetIds).distinct val utegQuery = (tweetIdsToSend, (query.getRequiredUserId, seedUserWeights)) client(utegQuery).map(handleResponse(candidates, _)) } private def handleResponse( candidates: Seq[CandidateWithFeatures[TweetCandidate]], results: KeyValueResult[Long, uteg.TweetRecommendation], ): Seq[FeatureMap] = { candidates.map { candidate => val inNetwork = candidate.features.getOrElse(FromInNetworkSourceFeature, false) val candidateProof = results(candidate.candidate.id).toOption.flatten val sourceProof = candidate.features .getOrElse(SourceTweetIdFeature, None).flatMap(results(_).toOption.flatten) val proofs = Seq(candidateProof, sourceProof).flatten.map(_.socialProofByType) val favoritedBy = proofs.flatMap(_.get(rc.SocialProofType.Favorite)).flatten val retweetedBy = proofs.flatMap(_.get(rc.SocialProofType.Retweet)).flatten val repliedBy = proofs.flatMap(_.get(rc.SocialProofType.Reply)).flatten val (favoritedByCount, retweetedByCount, repliedByCount) = if (!inNetwork) { (favoritedBy.size.toDouble, retweetedBy.size.toDouble, repliedBy.size.toDouble) } else { (0.0, 0.0, 0.0) } FeatureMapBuilder(sizeHint = 6) .add(FavoritedByUserIdsFeature, favoritedBy) .add(RetweetedByEngagerIdsFeature, retweetedBy) .add(RepliedByEngagerIdsFeature, repliedBy) .add(FavoritedByCountFeature, favoritedByCount) .add(RetweetedByCountFeature, retweetedByCount) .add(RepliedByCountFeature, repliedByCount) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/VideoSummaryEmbeddingFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content.VideoSummaryEmbeddingFeaturesAdaptor import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableVideoSummaryEmbeddingFeatureDeciderParam import com.twitter.home_mixer.param.HomeMixerInjectionNames.VideoEmbeddingMHStore import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.storehaus.ReadableStore import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import com.twitter.media_understanding.video_summary.thriftscala.VideoEmbedding import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.util.logging.Logging import scala.collection.JavaConverters._ object VideoSummaryEmbeddingFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class VideoSummaryEmbeddingFeatureHydrator @Inject() ( @Named(VideoEmbeddingMHStore) videoEmbeddingStore: ReadableStore[Long, VideoEmbedding], statsReceiver: StatsReceiver) extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] with Logging { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "VideoSummaryEmbedding") override val features: Set[Feature[_, _]] = Set(VideoSummaryEmbeddingFeature) private val DefaultFeatureMap = FeatureMapBuilder().add(VideoSummaryEmbeddingFeature, new DataRecord()).build() private val scopedCounter = statsReceiver.scope("VideoSummaryEmbeddingHydration") private val nonEmptyCounter = scopedCounter.counter("nonEmpty") private val emptyCounter = scopedCounter.counter("empty") override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableVideoSummaryEmbeddingFeatureDeciderParam) override def apply( query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap ): Stitch[FeatureMap] = { getFirstMediaId(existingFeatures).fold(Stitch.value(DefaultFeatureMap)) { mediaId => Stitch .callFuture(videoEmbeddingStore.get(mediaId)).map { resultOpt => resultOpt match { case Some(embedding) => val dataRecord = VideoSummaryEmbeddingFeaturesAdaptor .adaptToDataRecords(embedding.embedding).asScala.head nonEmptyCounter.incr() FeatureMapBuilder().add(VideoSummaryEmbeddingFeature, dataRecord).build() case None => emptyCounter.incr() DefaultFeatureMap } }.onFailure { e => error(s"Error fetching VideoSummaryEmbedding: $e") Stitch.value(DefaultFeatureMap) } } } private def getFirstMediaId(featureMap: FeatureMap): Option[Long] = featureMap.getOrElse(TweetMediaIdsFeature, Seq.empty[Long]).headOption } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ViewCountsFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.ViewCountFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.strato.catalog.Fetch import com.twitter.strato.generated.client.viewcounts.ViewCountOnTweetClientColumn import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton @Singleton class ViewCountsFeatureHydrator @Inject() ( viewCountsColumn: ViewCountOnTweetClientColumn, statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ViewCounts") override val features: Set[Feature[_, _]] = Set(ViewCountFeature) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") private val DefaultFeatureMap = FeatureMap(ViewCountFeature, None) private val batchSize = 64 def getFeatureMaps( candidates: Seq[CandidateWithFeatures[TweetCandidate]], ): Future[Seq[FeatureMap]] = { val featureMapStitch = Stitch.traverse(candidates) { candidate => viewCountsColumn.fetcher .fetch(candidate.candidate.id, Unit) .map { case Fetch.Result(response, _) => if (response.nonEmpty) keyFoundCounter.incr() else keyNotFoundCounter.incr() FeatureMap(ViewCountFeature, response) case _ => keyFailureCounter.incr() DefaultFeatureMap } } Stitch.run(featureMapStitch) } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { OffloadFuturePools.offloadBatchSeqToFutureSeq( candidates, getFeatureMaps, batchSize, offload = true) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ViralContentCreatorMetricsFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.ViralContentCreatorFeature import com.twitter.home_mixer.module.ViralContentCreatorsConfig import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton /** * Track metrics on how often we serve posts from viral content creators */ @Singleton case class ViralContentCreatorMetricsFeatureHydrator @Inject() ( viralContentCreatorsConfig: ViralContentCreatorsConfig) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ViralContentCreatorMetrics") override val features: Set[Feature[_, _]] = Set(ViralContentCreatorFeature) private val ViralContentCreators = viralContentCreatorsConfig.creators override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { candidates.map { candidate => val authorIdOpt = candidate.features.getOrElse(AuthorIdFeature, None) FeatureMap(ViralContentCreatorFeature, authorIdOpt.exists(ViralContentCreators.contains)) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/WithDefaultFeatureMap.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator import com.twitter.product_mixer.core.feature.featuremap.FeatureMap trait WithDefaultFeatureMap { // Make sure that default Feature Map has same features as defined in the feature hydrator val defaultFeatureMap: FeatureMap } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features import com.twitter.home_mixer.util.DataRecordUtil import com.twitter.ml.api.DataRecord import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.util.CompactDataRecordConverter import com.twitter.ml.api.util.FDsl._ import com.twitter.timelines.author_features.v1.{thriftjava => af} import com.twitter.timelines.prediction.common.adapters.TimelinesAdapterBase import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures import scala.collection.JavaConverters._ object AuthorFeaturesAdapter extends TimelinesAdapterBase[af.AuthorFeatures] { private val Prefix = "original_author.timelines.original_author_aggregates." private val typedAggregateGroups = TimelinesAggregationConfig.originalAuthorAggregatesV1.buildTypedAggregateGroups() private val aggregateFeaturesRenameMap: Map[Feature[_], Feature[_]] = typedAggregateGroups.map(_.outputFeaturesToRenamedOutputFeatures(Prefix)).reduce(_ ++ _) private val prefixedOriginalAuthorAggregateFeatures = typedAggregateGroups.flatMap(_.allOutputFeatures).map { feature => aggregateFeaturesRenameMap.getOrElse(feature, feature) } private val authorFeatures = prefixedOriginalAuthorAggregateFeatures ++ Seq( UserHealthFeatures.AuthorState, UserHealthFeatures.NumAuthorFollowers, UserHealthFeatures.NumAuthorConnectDays, UserHealthFeatures.NumAuthorConnect ) private val aggregateFeatureContext: FeatureContext = new FeatureContext(typedAggregateGroups.flatMap(_.allOutputFeatures).asJava) private lazy val prefixedAggregateFeatureContext: FeatureContext = new FeatureContext(prefixedOriginalAuthorAggregateFeatures.asJava) override val getFeatureContext: FeatureContext = new FeatureContext(authorFeatures: _*) override val commonFeatures: Set[Feature[_]] = Set.empty private val compactDataRecordConverter = new CompactDataRecordConverter() override def adaptToDataRecords( authorFeatures: af.AuthorFeatures ): java.util.List[DataRecord] = { val dataRecord = if (authorFeatures.aggregates != null) { val originalAuthorAggregatesDataRecord = compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates) DataRecordUtil.applyRename( originalAuthorAggregatesDataRecord, aggregateFeatureContext, prefixedAggregateFeatureContext, aggregateFeaturesRenameMap) } else new DataRecord if (authorFeatures.user_health != null) { val userHealth = authorFeatures.user_health if (userHealth.user_state != null) { dataRecord.setFeatureValue( UserHealthFeatures.AuthorState, userHealth.user_state.getValue.toLong ) } dataRecord.setFeatureValue( UserHealthFeatures.NumAuthorFollowers, userHealth.num_followers.toDouble ) dataRecord.setFeatureValue( UserHealthFeatures.NumAuthorConnectDays, userHealth.num_connect_days.toDouble ) dataRecord.setFeatureValue( UserHealthFeatures.NumAuthorConnect, userHealth.num_connect.toDouble ) } List(dataRecord).asJava } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "src/java/com/twitter/ml/api:api-base", "src/java/com/twitter/ml/api/util", "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/scala/com/twitter/timelines/prediction/common/aggregates", "src/scala/com/twitter/timelines/prediction/features/user_health", "src/thrift/com/twitter/timelines/author_features:thrift-java", "timelines/data_processing/ml_util/aggregation_framework", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/timelines/prediction/common/adapters", "src/scala/com/twitter/timelines/prediction/features/common", "src/scala/com/twitter/timelines/prediction/features/conversation_features", "src/scala/com/twitter/timelines/prediction/features/recap", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/ClipEmbeddingFeaturesAdapter.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.FloatTensor import com.twitter.ml.api.GeneralTensor import com.twitter.ml.api.RichDataRecord import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import scala.collection.JavaConverters._ object ClipEmbeddingFeaturesAdapter extends TimelinesMutatingAdapterBase[Seq[Double]] { val ClipEmbeddingsFeature: Feature.Tensor = TimelinesSharedFeatures.CLIP_EMBEDDING override val getFeatureContext: FeatureContext = new FeatureContext(ClipEmbeddingsFeature) override val commonFeatures: Set[Feature[_]] = Set.empty override def setFeatures( clipEmbedding: Seq[Double], richDataRecord: RichDataRecord ): Unit = { val clipEmbeddingTensor = new GeneralTensor() clipEmbeddingTensor.setFloatTensor(new FloatTensor(clipEmbedding.map(Double.box).asJava)) richDataRecord.setFeatureValue( ClipEmbeddingsFeature, clipEmbeddingTensor ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/ContentFeatureAdapter.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content import com.twitter.home_mixer.model.ContentFeatures import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.ml.api.util.DataRecordConverters._ import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.common.adapters.TweetLengthType import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import com.twitter.timelines.prediction.features.conversation_features.ConversationFeatures import com.twitter.timelines.prediction.features.recap.RecapFeatures import scala.collection.JavaConverters._ object ContentFeatureAdapter extends TimelinesMutatingAdapterBase[Option[ContentFeatures]] { override val getFeatureContext: FeatureContext = new FeatureContext( ConversationFeatures.IS_SELF_THREAD_TWEET, ConversationFeatures.IS_LEAF_IN_SELF_THREAD, TimelinesSharedFeatures.ASPECT_RATIO_DEN, TimelinesSharedFeatures.ASPECT_RATIO_NUM, TimelinesSharedFeatures.BIT_RATE, TimelinesSharedFeatures.CLASSIFICATION_LABELS, TimelinesSharedFeatures.COLOR_1_BLUE, TimelinesSharedFeatures.COLOR_1_GREEN, TimelinesSharedFeatures.COLOR_1_PERCENTAGE, TimelinesSharedFeatures.COLOR_1_RED, TimelinesSharedFeatures.FACE_AREAS, TimelinesSharedFeatures.HAS_APP_INSTALL_CALL_TO_ACTION, TimelinesSharedFeatures.HAS_DESCRIPTION, TimelinesSharedFeatures.HAS_QUESTION, TimelinesSharedFeatures.HAS_SELECTED_PREVIEW_IMAGE, TimelinesSharedFeatures.HAS_TITLE, TimelinesSharedFeatures.HAS_VISIT_SITE_CALL_TO_ACTION, TimelinesSharedFeatures.HAS_WATCH_NOW_CALL_TO_ACTION, TimelinesSharedFeatures.HEIGHT_1, TimelinesSharedFeatures.HEIGHT_2, TimelinesSharedFeatures.HEIGHT_3, TimelinesSharedFeatures.HEIGHT_4, TimelinesSharedFeatures.IS_360, TimelinesSharedFeatures.IS_EMBEDDABLE, TimelinesSharedFeatures.IS_MANAGED, TimelinesSharedFeatures.IS_MONETIZABLE, TimelinesSharedFeatures.MEDIA_PROVIDERS, TimelinesSharedFeatures.NUM_CAPS, TimelinesSharedFeatures.NUM_COLOR_PALLETTE_ITEMS, TimelinesSharedFeatures.NUM_FACES, TimelinesSharedFeatures.NUM_MEDIA_TAGS, TimelinesSharedFeatures.NUM_NEWLINES, TimelinesSharedFeatures.NUM_STICKERS, TimelinesSharedFeatures.NUM_WHITESPACES, TimelinesSharedFeatures.RESIZE_METHOD_1, TimelinesSharedFeatures.RESIZE_METHOD_2, TimelinesSharedFeatures.RESIZE_METHOD_3, TimelinesSharedFeatures.RESIZE_METHOD_4, TimelinesSharedFeatures.TWEET_LENGTH, TimelinesSharedFeatures.TWEET_LENGTH_TYPE, TimelinesSharedFeatures.VIDEO_DURATION, TimelinesSharedFeatures.VIEW_COUNT, TimelinesSharedFeatures.WIDTH_1, TimelinesSharedFeatures.WIDTH_2, TimelinesSharedFeatures.WIDTH_3, TimelinesSharedFeatures.WIDTH_4, RecapFeatures.HAS_VIDEO, RecapFeatures.HAS_IMAGE, ) override val commonFeatures: Set[Feature[_]] = Set.empty private def getTweetLengthType(tweetLength: Int): Long = { tweetLength match { case x if 0 > x || 280 < x => TweetLengthType.INVALID case x if 0 <= x && x <= 30 => TweetLengthType.VERY_SHORT case x if 30 < x && x <= 60 => TweetLengthType.SHORT case x if 60 < x && x <= 90 => TweetLengthType.MEDIUM case x if 90 < x && x <= 140 => TweetLengthType.LENGTHY case x if 140 < x && x <= 210 => TweetLengthType.VERY_LENGTHY case x if x > 210 => TweetLengthType.MAXIMUM_LENGTH } } override def setFeatures( contentFeatures: Option[ContentFeatures], richDataRecord: RichDataRecord ): Unit = { if (contentFeatures.nonEmpty) { val features = contentFeatures.get // Conversation Features richDataRecord.setFeatureValueFromOption( ConversationFeatures.IS_SELF_THREAD_TWEET, Some(features.selfThreadMetadata.nonEmpty) ) richDataRecord.setFeatureValueFromOption( ConversationFeatures.IS_LEAF_IN_SELF_THREAD, features.selfThreadMetadata.map(_.isLeaf) ) // Media Features richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.ASPECT_RATIO_DEN, features.aspectRatioDen.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.ASPECT_RATIO_NUM, features.aspectRatioNum.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.BIT_RATE, features.bitRate.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HEIGHT_1, features.heights.flatMap(_.lift(0)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HEIGHT_2, features.heights.flatMap(_.lift(1)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HEIGHT_3, features.heights.flatMap(_.lift(2)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HEIGHT_4, features.heights.flatMap(_.lift(3)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.NUM_MEDIA_TAGS, features.numMediaTags.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.RESIZE_METHOD_1, features.resizeMethods.flatMap(_.lift(0)).map(_.toLong) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.RESIZE_METHOD_2, features.resizeMethods.flatMap(_.lift(1)).map(_.toLong) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.RESIZE_METHOD_3, features.resizeMethods.flatMap(_.lift(2)).map(_.toLong) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.RESIZE_METHOD_4, features.resizeMethods.flatMap(_.lift(3)).map(_.toLong) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.VIDEO_DURATION, features.videoDurationMs.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.WIDTH_1, features.widths.flatMap(_.lift(0)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.WIDTH_2, features.widths.flatMap(_.lift(1)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.WIDTH_3, features.widths.flatMap(_.lift(2)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.WIDTH_4, features.widths.flatMap(_.lift(3)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.NUM_COLOR_PALLETTE_ITEMS, features.numColors.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.COLOR_1_RED, features.dominantColorRed.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.COLOR_1_BLUE, features.dominantColorBlue.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.COLOR_1_GREEN, features.dominantColorGreen.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.COLOR_1_PERCENTAGE, features.dominantColorPercentage ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.MEDIA_PROVIDERS, features.mediaOriginProviders.map(_.toSet.asJava) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.IS_360, features.is360 ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.VIEW_COUNT, features.viewCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.IS_MANAGED, features.isManaged ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.IS_MONETIZABLE, features.isMonetizable ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.IS_EMBEDDABLE, features.isEmbeddable ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.NUM_STICKERS, features.stickerIds.map(_.length.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.NUM_FACES, features.faceAreas.map(_.length.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.FACE_AREAS, // guard for exception from max on empty seq features.faceAreas.map(faceAreas => faceAreas.map(_.toDouble).reduceOption(_ max _).getOrElse(0.0)) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HAS_SELECTED_PREVIEW_IMAGE, features.hasSelectedPreviewImage ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HAS_TITLE, features.hasTitle ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HAS_DESCRIPTION, features.hasDescription ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HAS_VISIT_SITE_CALL_TO_ACTION, features.hasVisitSiteCallToAction ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HAS_APP_INSTALL_CALL_TO_ACTION, features.hasAppInstallCallToAction ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HAS_WATCH_NOW_CALL_TO_ACTION, features.hasWatchNowCallToAction ) // text features richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.NUM_CAPS, Some(features.numCaps.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.TWEET_LENGTH, Some(features.length.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.TWEET_LENGTH_TYPE, Some(getTweetLengthType(features.length.toInt)) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.NUM_WHITESPACES, Some(features.numWhiteSpaces.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HAS_QUESTION, Some(features.hasQuestion) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.NUM_NEWLINES, features.numNewlines.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( RecapFeatures.HAS_IMAGE, features.hasImage ) richDataRecord.setFeatureValueFromOption( RecapFeatures.HAS_VIDEO, features.hasVideo ) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/TextTokensFeaturesAdapter.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.ml.api.thriftscala.Int32Tensor import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import com.twitter.ml.api.{thriftscala => ml} object TextTokensFeaturesAdapter extends TimelinesMutatingAdapterBase[Seq[Int]] { private val TextTokenFeature: Feature.Tensor = TimelinesSharedFeatures.TEXT_TOKENS_EMBEDDING override val getFeatureContext: FeatureContext = new FeatureContext(TextTokenFeature) override val commonFeatures: Set[Feature[_]] = Set.empty override def setFeatures( textTokens: Seq[Int], richDataRecord: RichDataRecord ): Unit = { richDataRecord.setFeatureValue( TextTokenFeature, ScalaToJavaDataRecordConversions.scalaTensor2Java( ml.GeneralTensor.Int32Tensor(Int32Tensor(ints = textTokens)) )) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content/VideoSummaryEmbeddingFeaturesAdaptor.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.FloatTensor import com.twitter.ml.api.RichDataRecord import com.twitter.ml.api.GeneralTensor import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import scala.collection.JavaConverters._ object VideoSummaryEmbeddingFeaturesAdaptor extends TimelinesMutatingAdapterBase[Seq[Double]] { val VideoSummaryEmbeddingFeature: Feature.Tensor = TimelinesSharedFeatures.VIDEO_SUMMARY_EMBEDDING override val getFeatureContext: FeatureContext = new FeatureContext(VideoSummaryEmbeddingFeature) override val commonFeatures: Set[Feature[_]] = Set.empty override def setFeatures( videoSummaryEmbedding: Seq[Double], richDataRecord: RichDataRecord ): Unit = { val summaryEmbeddingTensor = new GeneralTensor() summaryEmbeddingTensor.setFloatTensor( new FloatTensor(videoSummaryEmbedding.map(Double.box).asJava)) richDataRecord.setFeatureValue( VideoSummaryEmbeddingFeature, summaryEmbeddingTensor ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/gizmoduck_features/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/scala/com/twitter/timelines/prediction/features/gizmoduck", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/gizmoduck_features/GizmoduckFeaturesAdapter.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.gizmoduck_features import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.features.gizmoduck.GizmoduckFeatures import java.lang.{Boolean => JBoolean} object GizmoduckFeaturesAdapter extends TimelinesMutatingAdapterBase[GFeatures] { override val getFeatureContext: FeatureContext = new FeatureContext( GizmoduckFeatures.AUTHOR_IS_BLUE_VERIFIED, GizmoduckFeatures.AUTHOR_IS_VERIFIED_ORGANIZATION, GizmoduckFeatures.AUTHOR_IS_VERIFIED_ORGANIZATION_AFFILIATE, GizmoduckFeatures.AUTHOR_IS_PROTECTED, ) override val commonFeatures: Set[Feature[_]] = Set.empty override def candidateFeatures: Set[Feature[_]] = super.candidateFeatures override def setFeatures( contentFeatures: GFeatures, richDataRecord: RichDataRecord ): Unit = { richDataRecord.setFeatureValue[JBoolean]( GizmoduckFeatures.AUTHOR_IS_BLUE_VERIFIED, contentFeatures.isBlueVerified ) richDataRecord.setFeatureValue[JBoolean]( GizmoduckFeatures.AUTHOR_IS_VERIFIED_ORGANIZATION, contentFeatures.isVerifiedOrganization ) richDataRecord.setFeatureValue[JBoolean]( GizmoduckFeatures.AUTHOR_IS_VERIFIED_ORGANIZATION_AFFILIATE, contentFeatures.isVerifiedOrganizationAffiliate ) richDataRecord.setFeatureValue[JBoolean]( GizmoduckFeatures.AUTHOR_IS_PROTECTED, contentFeatures.isProtected ) } } trait GFeatures { def isBlueVerified: Boolean def isVerifiedOrganization: Boolean def isVerifiedOrganizationAffiliate: Boolean def isProtected: Boolean } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/scala/com/twitter/timelines/prediction/features/common", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import scala.collection.JavaConverters._ object InferredTopicAdapter extends TimelinesMutatingAdapterBase[Map[Long, Double]] { override val getFeatureContext: FeatureContext = new FeatureContext( TimelinesSharedFeatures.INFERRED_TOPIC_IDS) override val commonFeatures: Set[Feature[_]] = Set.empty override def setFeatures( inferredTopicFeatures: Map[Long, Double], richDataRecord: RichDataRecord ): Unit = { richDataRecord.setFeatureValue( TimelinesSharedFeatures.INFERRED_TOPIC_IDS, inferredTopicFeatures.keys.map(_.toString).toSet.asJava) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/light_ranking_features/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/thrift/src/main/thrift:thrift-scala", "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/scala/com/twitter/timelines/prediction/features/common", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/light_ranking_features/LightRankingCandidateFeaturesAdapter.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.light_ranking_features import com.twitter.home_mixer.thriftscala.ServedType import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import java.lang.{Boolean => JBoolean} import java.lang.{Long => JLong} import java.lang.{String => JString} case class LightRankingCandidateFeatures( isSelected: Boolean, isSelectedByHeavyRanker: Boolean, rankByHeavyRanker: Int, servedType: ServedType, candidateSourcePosition: Long, ) /** * define light ranking features adapter to create a data record which includes many light ranking features * e.g. predictionRequestId, userId, tweetId to be used as joined key in batch pipeline. */ object LightRankingCandidateFeaturesAdapter extends TimelinesMutatingAdapterBase[LightRankingCandidateFeatures] { val featureContext = new FeatureContext( TimelinesSharedFeatures.IS_SELECTED, TimelinesSharedFeatures.IS_SELECTED_BY_HEAVY_RANKER, TimelinesSharedFeatures.RANK_BY_HEAVY_RANKER, TimelinesSharedFeatures.SERVED_TYPE_ID, TimelinesSharedFeatures.SERVED_TYPE, TimelinesSharedFeatures.CANDIDATE_SOURCE_POSITION, ) override def getFeatureContext: FeatureContext = featureContext override val commonFeatures: Set[Feature[_]] = Set.empty override def setFeatures( lightRankingCandidateFeatures: LightRankingCandidateFeatures, richDataRecord: RichDataRecord ): Unit = { richDataRecord.setFeatureValue[JBoolean]( TimelinesSharedFeatures.IS_SELECTED, lightRankingCandidateFeatures.isSelected) richDataRecord.setFeatureValue[JBoolean]( TimelinesSharedFeatures.IS_SELECTED_BY_HEAVY_RANKER, lightRankingCandidateFeatures.isSelectedByHeavyRanker) richDataRecord.setFeatureValue[JLong]( TimelinesSharedFeatures.RANK_BY_HEAVY_RANKER, lightRankingCandidateFeatures.rankByHeavyRanker) richDataRecord.setFeatureValue[JString]( TimelinesSharedFeatures.SERVED_TYPE, lightRankingCandidateFeatures.servedType.name) richDataRecord.setFeatureValue[JLong]( TimelinesSharedFeatures.SERVED_TYPE_ID, lightRankingCandidateFeatures.servedType.getValue()) richDataRecord.setFeatureValue[JLong]( TimelinesSharedFeatures.CANDIDATE_SOURCE_POSITION, lightRankingCandidateFeatures.candidateSourcePosition) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "src/java/com/twitter/ml/api:api-base", "src/java/com/twitter/ml/api/constant", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/scala/com/twitter/timelines/prediction/features/common", "src/scala/com/twitter/timelines/prediction/features/request_context", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features import com.twitter.ml.api.constant.SharedFeatures import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import java.lang.{Long => JLong} case class NonMLCandidateFeatures( tweetId: Long, sourceTweetId: Long, originalAuthorId: Option[Long], ) /** * define non ml features adapter to create a data record which includes many non ml features * e.g. predictionRequestId, userId, tweetId to be used as joined key in batch pipeline. */ object NonMLCandidateFeaturesAdapter extends TimelinesMutatingAdapterBase[NonMLCandidateFeatures] { private val featureContext = new FeatureContext( SharedFeatures.TWEET_ID, // For Secondary Engagement data generation TimelinesSharedFeatures.SOURCE_TWEET_ID, TimelinesSharedFeatures.ORIGINAL_AUTHOR_ID, ) override def getFeatureContext: FeatureContext = featureContext override val commonFeatures: Set[Feature[_]] = Set.empty override def setFeatures( nonMLCandidateFeatures: NonMLCandidateFeatures, richDataRecord: RichDataRecord ): Unit = { richDataRecord.setFeatureValue[JLong](SharedFeatures.TWEET_ID, nonMLCandidateFeatures.tweetId) richDataRecord.setFeatureValue[JLong]( TimelinesSharedFeatures.SOURCE_TWEET_ID, nonMLCandidateFeatures.sourceTweetId) nonMLCandidateFeatures.originalAuthorId.foreach( richDataRecord.setFeatureValue[JLong](TimelinesSharedFeatures.ORIGINAL_AUTHOR_ID, _)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.ml.api.constant.SharedFeatures import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import com.twitter.timelines.prediction.features.request_context.RequestContextFeatures import java.lang.{Long => JLong} import java.lang.{String => JString} case class NonMLCommonFeatures( userId: Long, guestId: Option[Long], clientId: Option[Long], countryCode: Option[String], predictionRequestId: Option[Long], productSurface: String, servedTimestamp: Long, ) /** * define non ml features adapter to create a data record which includes many non ml features * e.g. predictionRequestId, userId, tweetId to be used as joined key in batch pipeline. */ object NonMLCommonFeaturesAdapter extends TimelinesMutatingAdapterBase[NonMLCommonFeatures] { private val featureContext = new FeatureContext( SharedFeatures.USER_ID, SharedFeatures.GUEST_ID, SharedFeatures.CLIENT_ID, TimelinesSharedFeatures.PREDICTION_REQUEST_ID, TimelinesSharedFeatures.PRODUCT_SURFACE, TimelinesSharedFeatures.SERVED_TIMESTAMP, RequestContextFeatures.COUNTRY_CODE, ) override def getFeatureContext: FeatureContext = featureContext override val commonFeatures: Set[Feature[_]] = Set( SharedFeatures.USER_ID, SharedFeatures.GUEST_ID, SharedFeatures.CLIENT_ID, TimelinesSharedFeatures.PREDICTION_REQUEST_ID, TimelinesSharedFeatures.PRODUCT_SURFACE, TimelinesSharedFeatures.SERVED_TIMESTAMP, RequestContextFeatures.COUNTRY_CODE, ) override def setFeatures( nonMLCommonFeatures: NonMLCommonFeatures, richDataRecord: RichDataRecord ): Unit = { richDataRecord.setFeatureValue[JLong](SharedFeatures.USER_ID, nonMLCommonFeatures.userId) nonMLCommonFeatures.guestId.foreach( richDataRecord.setFeatureValue[JLong](SharedFeatures.GUEST_ID, _)) nonMLCommonFeatures.clientId.foreach( richDataRecord.setFeatureValue[JLong](SharedFeatures.CLIENT_ID, _)) nonMLCommonFeatures.predictionRequestId.foreach( richDataRecord.setFeatureValue[JLong](TimelinesSharedFeatures.PREDICTION_REQUEST_ID, _)) nonMLCommonFeatures.countryCode.foreach( richDataRecord.setFeatureValue[JString](RequestContextFeatures.COUNTRY_CODE, _)) richDataRecord.setFeatureValue( TimelinesSharedFeatures.PRODUCT_SURFACE, nonMLCommonFeatures.productSurface) richDataRecord.setFeatureValue[JLong]( TimelinesSharedFeatures.SERVED_TIMESTAMP, nonMLCommonFeatures.servedTimestamp) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/thrift/com/twitter/ml/api:data-java", "timelines/data_processing/ml_util/aggregation_framework/conversion:for-timelines", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.offline_aggregates import com.twitter.ml.api.DataRecord import com.twitter.ml.api.IRecordOneToOneAdapter object PassThroughAdapter extends IRecordOneToOneAdapter[Seq[DataRecord]] { override def adaptToDataRecord(record: Seq[DataRecord]): DataRecord = record.headOption.getOrElse(new DataRecord) // This is not necessary and should not be used. override def getFeatureContext = ??? } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.offline_aggregates import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.timelines.data_processing.ml_util.aggregation_framework.conversion.CombineCountsPolicy import com.twitter.timelines.prediction.common.adapters.TimelinesIRecordAdapter class SparseAggregatesToDenseAdapter(policy: CombineCountsPolicy) extends TimelinesIRecordAdapter[Seq[DataRecord]] { override def setFeatures(input: Seq[DataRecord], mutableDataRecord: RichDataRecord): Unit = policy.defaultMergeRecord(mutableDataRecord.getRecord, input) override val getFeatureContext: FeatureContext = new FeatureContext(policy.outputFeaturesPostMerge.toSeq: _*) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/simclusters_features/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/timelines/prediction/common/adapters", "src/scala/com/twitter/timelines/prediction/features/simcluster", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/simclusters_features/SimclustersFeaturesAdapter.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.simclusters_features import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.simclusters_v2.thriftscala.SimClustersEmbedding import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.features.simcluster.SimclusterFeatures import java.util import java.util.Collections object SimclustersFeaturesAdapter extends TimelinesMutatingAdapterBase[Option[SimClustersEmbedding]] { val SimclustersSparseTweetEmbeddingsFeature: Feature.SparseContinuous = SimclusterFeatures.SIMCLUSTER_TWEET_CLUSTER_SCORES override val getFeatureContext: FeatureContext = new FeatureContext( SimclustersSparseTweetEmbeddingsFeature) override val commonFeatures: Set[Feature[_]] = Set.empty override def setFeatures( simClustersEmbedding: Option[SimClustersEmbedding], richDataRecord: RichDataRecord ): Unit = { val simclustersWithScoresMap = simClustersEmbedding match { case Some(value) => val result = new util.HashMap[String, java.lang.Double](value.embedding.size, 1.0f) value.embedding.foreach { simclusterWithScore => result.put(simclusterWithScore.clusterId.toString, simclusterWithScore.score) } result case None => Collections.emptyMap[String, java.lang.Double]() } richDataRecord.setFeatureValue( SimclustersSparseTweetEmbeddingsFeature, simclustersWithScoresMap ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/thrift/com/twitter/ml/api:data-java", "user_history_transformer/thrift/src/main/thrift/com/twitter/user_history_transformer:user_history_transformer-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/TransformerEmbeddingsAdapter.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings import com.twitter.ml.api.DataType import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions import com.twitter.ml.api.{thriftscala => ml} import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import scala.collection.JavaConverters._ sealed trait TransformerEmbeddingsAdapter extends TimelinesMutatingAdapterBase[Option[ml.FloatTensor]] { def embeddingsFeature: Feature.Tensor override def getFeatureContext: FeatureContext = new FeatureContext( embeddingsFeature ) override def setFeatures( embedding: Option[ml.FloatTensor], richDataRecord: RichDataRecord ): Unit = { embedding.foreach { floatTensor => richDataRecord.setFeatureValue( embeddingsFeature, ScalaToJavaDataRecordConversions.scalaTensor2Java( ml.GeneralTensor .FloatTensor(floatTensor))) } } } sealed trait TransformerByteEmbeddingsAdapter extends TimelinesMutatingAdapterBase[Option[ml.RawTypedTensor]] { def embeddingsFeature: Feature.Tensor override def getFeatureContext: FeatureContext = new FeatureContext( embeddingsFeature ) override def setFeatures( embedding: Option[ml.RawTypedTensor], richDataRecord: RichDataRecord ): Unit = { embedding.foreach { tensor => richDataRecord.setFeatureValue( embeddingsFeature, ScalaToJavaDataRecordConversions.scalaTensor2Java( ml.GeneralTensor .RawTypedTensor(tensor))) } } } object TransformerEmbeddingsFeatures { val UserHistoryTransformerEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( "user.transformer.user_history_as_float_tensor", DataType.FLOAT, List(128L).map(long2Long).asJava, ) val UserHistoryTransformerEmbeddingsGreenFeature: Feature.Tensor = new Feature.Tensor( "user.transformer.green.user_history_as_float_tensor", DataType.BYTE, List(1024L).map(long2Long).asJava, ) val UserHistoryTransformerEmbeddingsJointBlueFeature: Feature.Tensor = new Feature.Tensor( "user.transformer.joint.blue.user_history_as_float_tensor", DataType.BYTE, List(1024L).map(long2Long).asJava, ) val PostTransformerEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( "tweet_id.transformer.post_as_float_tensor", DataType.FLOAT, List(128L).map(long2Long).asJava, ) val PostTransformerEmbeddingsGreenFeature: Feature.Tensor = new Feature.Tensor( "tweet_id.transformer.green.post_as_float_tensor", DataType.FLOAT, List(128L).map(long2Long).asJava, ) val PostTransformerEmbeddingsJointBlueFeature: Feature.Tensor = new Feature.Tensor( "tweet_id.transformer.joint.blue.post_as_float_tensor", DataType.FLOAT, List(128L).map(long2Long).asJava, ) } object UserHistoryTransformerEmbeddingsHomeBlueAdapter extends TransformerEmbeddingsAdapter { override val embeddingsFeature: Feature.Tensor = TransformerEmbeddingsFeatures.UserHistoryTransformerEmbeddingsFeature override val commonFeatures: Set[Feature[_]] = Set.empty } object UserHistoryTransformerEmbeddingsHomeGreenAdapter extends TransformerByteEmbeddingsAdapter { override val embeddingsFeature: Feature.Tensor = TransformerEmbeddingsFeatures.UserHistoryTransformerEmbeddingsGreenFeature override val commonFeatures: Set[Feature[_]] = Set.empty } object UserHistoryTransformerEmbeddingsJointBlueAdapter extends TransformerByteEmbeddingsAdapter { override val embeddingsFeature: Feature.Tensor = TransformerEmbeddingsFeatures.UserHistoryTransformerEmbeddingsJointBlueFeature override val commonFeatures: Set[Feature[_]] = Set.empty } object PostTransformerEmbeddingsHomeBlueAdapter extends TransformerEmbeddingsAdapter { override val embeddingsFeature: Feature.Tensor = TransformerEmbeddingsFeatures.PostTransformerEmbeddingsFeature override val commonFeatures: Set[Feature[_]] = Set.empty } object PostTransformerEmbeddingsHomeGreenAdapter extends TransformerEmbeddingsAdapter { override val embeddingsFeature: Feature.Tensor = TransformerEmbeddingsFeatures.PostTransformerEmbeddingsGreenFeature override val commonFeatures: Set[Feature[_]] = Set.empty } object PostTransformerEmbeddingsJointBlueAdapter extends TransformerEmbeddingsAdapter { override val embeddingsFeature: Feature.Tensor = TransformerEmbeddingsFeatures.PostTransformerEmbeddingsJointBlueFeature override val commonFeatures: Set[Feature[_]] = Set.empty } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/UserHistoryEventsAdapter.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings import com.twitter.ml.api.DataType import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.GeneralTensor import com.twitter.ml.api.Int32Tensor import com.twitter.ml.api.Int64Tensor import com.twitter.ml.api.RawTypedTensor import com.twitter.ml.api.RichDataRecord import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.user_history_transformer.thriftscala.UserHistory import scala.collection.JavaConverters._ import java.nio.ByteBuffer object UserHistoryEventsFeatures { val TweetIdsFeature: Feature.Tensor = new Feature.Tensor( "user.history.user_history_events_tweet_ids", DataType.INT64, List(100L).map(long2Long).asJava, ) val AuthorIdsFeature: Feature.Tensor = new Feature.Tensor( "user.history.user_history_events_author_ids", DataType.INT64, List(100L).map(long2Long).asJava, ) val ActionTypesFeature: Feature.Tensor = new Feature.Tensor( "user.history.user_history_events_action_types", DataType.INT32, List(100L).map(long2Long).asJava, ) val ActionTimestampsFeature: Feature.Tensor = new Feature.Tensor( "user.history.user_history_events_action_timestamps", DataType.INT64, List(100L).map(long2Long).asJava, ) val SemanticIdsFeature: Feature.Tensor = new Feature.Tensor( "user.history.user_history_events_semantic_ids", DataType.UINT8, List(500L).map(long2Long).asJava, ) } trait BaseUserHistoryEventsAdapter extends TimelinesMutatingAdapterBase[Seq[UserHistory]] { override val commonFeatures: Set[Feature[_]] = Set.empty private val DefaultSemanticId: Seq[Short] = Seq(0, 0, 0, 0, 0) def setAdditionalFeatures( record: Seq[UserHistory], mutableDataRecord: RichDataRecord ): Unit = {} override def setFeatures( record: Seq[UserHistory], mutableDataRecord: RichDataRecord ): Unit = { val tweetIdsFeature = GeneralTensor.int64Tensor( new Int64Tensor( record .map(history => history.sourceTweetId.getOrElse(history.tweetId)).map(long2Long).asJava)) val authorIdsFeature = GeneralTensor.int64Tensor( new Int64Tensor(record.map(_.authorId.getOrElse(-1L)).map(long2Long).asJava)) val actionTypesFeature = GeneralTensor.int32Tensor( new Int32Tensor(record.map(_.actionType.getValue()).map(int2Integer).asJava)) val actionTimestampsFeature = GeneralTensor.int64Tensor( new Int64Tensor(record.map(_.engagedTimestampMs).map(long2Long).asJava)) val semanticIdsFeature = GeneralTensor.rawTypedTensor( new RawTypedTensor( DataType.UINT8, ByteBuffer.wrap( record .flatMap(_.metadata.flatMap(_.semanticId).getOrElse(DefaultSemanticId)) .map(_.toByte).toArray) ) ) mutableDataRecord .setFeatureValue(UserHistoryEventsFeatures.TweetIdsFeature, tweetIdsFeature) mutableDataRecord .setFeatureValue(UserHistoryEventsFeatures.AuthorIdsFeature, authorIdsFeature) mutableDataRecord .setFeatureValue(UserHistoryEventsFeatures.ActionTypesFeature, actionTypesFeature) mutableDataRecord .setFeatureValue(UserHistoryEventsFeatures.ActionTimestampsFeature, actionTimestampsFeature) mutableDataRecord .setFeatureValue(UserHistoryEventsFeatures.SemanticIdsFeature, semanticIdsFeature) setAdditionalFeatures(record, mutableDataRecord) } override def getFeatureContext: FeatureContext = new FeatureContext( UserHistoryEventsFeatures.TweetIdsFeature, UserHistoryEventsFeatures.AuthorIdsFeature, UserHistoryEventsFeatures.ActionTypesFeature, UserHistoryEventsFeatures.ActionTimestampsFeature, UserHistoryEventsFeatures.SemanticIdsFeature, ) } object UserHistoryEventsAdapter extends BaseUserHistoryEventsAdapter {} ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings/VideoUserHistoryEventsAdapter.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings import com.twitter.ml.api.DataType import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.GeneralTensor import com.twitter.ml.api.Int32Tensor import com.twitter.ml.api.Int64Tensor import com.twitter.ml.api.RichDataRecord import com.twitter.ml.api.StringTensor import com.twitter.user_history_transformer.thriftscala.UserHistory import scala.collection.JavaConverters._ object VideoUserHistoryEventsFeatures { val VideoWatchTimesFeature: Feature.Tensor = new Feature.Tensor( "user.history.user_history_events_video_watch_times", DataType.INT64, List(100L).map(long2Long).asJava, ) val VideoMediaIdsFeature: Feature.Tensor = new Feature.Tensor( "user.history.user_history_events_video_media_ids", DataType.INT64, List(100L).map(long2Long).asJava, ) val VideoMediaCategoriesFeature: Feature.Tensor = new Feature.Tensor( "user.history.user_history_events_video_media_categories", DataType.INT32, List(100L).map(long2Long).asJava, ) val VideoDurationsFeature: Feature.Tensor = new Feature.Tensor( "user.history.user_history_events_video_durations", DataType.INT32, List(100L).map(long2Long).asJava, ) val VideoTopEntityFeature: Feature.Tensor = new Feature.Tensor( "user.history.user_history_events_video_top_entity_info", DataType.STRING, List(100L).map(long2Long).asJava, ) val VideoGrokTagsInfoFeature: Feature.Tensor = new Feature.Tensor( "user.history.user_history_events_video_grok_tags_info", DataType.STRING, List(100L).map(long2Long).asJava, ) val VideoCluster95IdsFeature: Feature.Tensor = new Feature.Tensor( "user.history.user_history_events_video_cluster95_ids_info", DataType.INT64, List(100L).map(long2Long).asJava, ) } object VideoUserHistoryEventsAdapter extends BaseUserHistoryEventsAdapter { override def getFeatureContext: FeatureContext = { super.getFeatureContext .addFeatures(VideoUserHistoryEventsFeatures.VideoWatchTimesFeature) .addFeatures(VideoUserHistoryEventsFeatures.VideoMediaIdsFeature) .addFeatures(VideoUserHistoryEventsFeatures.VideoMediaCategoriesFeature) .addFeatures(VideoUserHistoryEventsFeatures.VideoDurationsFeature) .addFeatures(VideoUserHistoryEventsFeatures.VideoCluster95IdsFeature) .addFeatures(VideoUserHistoryEventsFeatures.VideoTopEntityFeature) .addFeatures(VideoUserHistoryEventsFeatures.VideoGrokTagsInfoFeature) } override def setAdditionalFeatures( record: Seq[UserHistory], mutableDataRecord: RichDataRecord ): Unit = { val videoWatchTimes = new GeneralTensor() videoWatchTimes.setInt64Tensor( new Int64Tensor( record.flatMap(_.metadata.flatMap(_.watchTime).map(_.toLong)).map(long2Long).asJava) ) mutableDataRecord.setFeatureValue( VideoUserHistoryEventsFeatures.VideoWatchTimesFeature, videoWatchTimes ) val videoMediaIds = new GeneralTensor() videoMediaIds.setInt64Tensor( new Int64Tensor(record.flatMap(_.metadata.flatMap(_.mediaId)).map(long2Long).asJava) ) mutableDataRecord.setFeatureValue( VideoUserHistoryEventsFeatures.VideoMediaIdsFeature, videoMediaIds ) val videoMediaCategories = new GeneralTensor() videoMediaCategories.setInt32Tensor( new Int32Tensor( record.flatMap(_.metadata).flatMap(_.mediaCategory).map(_.value).map(int2Integer).asJava)) mutableDataRecord.setFeatureValue( VideoUserHistoryEventsFeatures.VideoMediaCategoriesFeature, videoMediaCategories ) val videoDurations = new GeneralTensor() videoDurations.setInt32Tensor( new Int32Tensor(record.flatMap(_.metadata).flatMap(_.videoDuration).map(int2Integer).asJava)) mutableDataRecord.setFeatureValue( VideoUserHistoryEventsFeatures.VideoDurationsFeature, videoDurations ) val topEntitiesInfo = new GeneralTensor() val topEntitiesAsString: Seq[String] = record.map { userHistory => userHistory.metadata .flatMap(_.entities) .map(_.filter(_.score.isDefined)) .filter(_.nonEmpty) .map(_.maxBy(_.score.get)) .headOption .map(entity => entity.qualifiedId._1.toString + ":" + entity.qualifiedId._2.toString) .getOrElse("") } topEntitiesInfo.setStringTensor(new StringTensor(topEntitiesAsString.asJava)) mutableDataRecord.setFeatureValue( VideoUserHistoryEventsFeatures.VideoTopEntityFeature, topEntitiesInfo ) val grokTagsInfo = new GeneralTensor() val grokTagsAsString: Seq[String] = record.map { userHistory => userHistory.metadata .flatMap(_.tags) .map(_.map(_.tag)) .getOrElse(Nil) .mkString(":") } grokTagsInfo.setStringTensor(new StringTensor(grokTagsAsString.asJava)) mutableDataRecord.setFeatureValue( VideoUserHistoryEventsFeatures.VideoGrokTagsInfoFeature, grokTagsInfo ) val clusterIdsFeature = new GeneralTensor() clusterIdsFeature.setInt64Tensor( new Int64Tensor(record.flatMap(_.metadata).flatMap(_.clusterId).map(long2Long).asJava)) mutableDataRecord.setFeatureValue( VideoUserHistoryEventsFeatures.VideoCluster95IdsFeature, clusterIdsFeature ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/thrift/com/twitter/ml/api:data-java", "src/thrift/com/twitter/ml/api:embedding-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings import com.twitter.ml.api.DataType import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions import com.twitter.ml.api.{thriftscala => ml} import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase sealed trait TwhinEmbeddingsAdapter extends TimelinesMutatingAdapterBase[Option[ml.FloatTensor]] { def twhinEmbeddingsFeature: Feature.Tensor override def getFeatureContext: FeatureContext = new FeatureContext( twhinEmbeddingsFeature ) override def setFeatures( embedding: Option[ml.FloatTensor], richDataRecord: RichDataRecord ): Unit = { embedding.foreach { floatTensor => richDataRecord.setFeatureValue( twhinEmbeddingsFeature, ScalaToJavaDataRecordConversions.scalaTensor2Java( ml.GeneralTensor .FloatTensor(floatTensor))) } } } object TwhinEmbeddingsFeatures { val TwhinAuthorFollowEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( "original_author.twhin.tw_hi_n.author_follow_as_float_tensor", DataType.FLOAT ) val TwhinUserEngagementEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( "user.twhin.tw_hi_n.user_engagement_as_float_tensor", DataType.FLOAT ) val TwhinRebuildUserEngagementEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( "user.twhin.tw_hi_n.rebuild_user_engagement_as_float_tensor", DataType.FLOAT ) val TwhinUserFollowEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( "user.twhin.tw_hi_n.user_follow_as_float_tensor", DataType.FLOAT ) val TwhinUserPositiveEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( "user.twhin.tw_hi_n.user_positive_as_float_tensor", DataType.FLOAT ) val TwhinRebuildUserPositiveEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( "user.twhin.tw_hi_n.rebuild_user_positive_as_float_tensor", DataType.FLOAT ) val TwhinUserNegativeEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( "user.twhin.tw_hi_n.user_negative_as_float_tensor", DataType.FLOAT ) val TwhinTweetEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( "original_tweet.twhin.tw_hi_n.tweet_v_2_as_float_tensor", DataType.FLOAT ) val TwhinRebuildTweetEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( "original_tweet.twhin.tw_hi_n.rebuild_tweet_as_float_tensor", DataType.FLOAT ) val TwhinVideoEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( "original_tweet.twhin.tw_hi_n.video_as_float_tensor", DataType.FLOAT ) } object TwhinAuthorFollowEmbeddingsAdapter extends TwhinEmbeddingsAdapter { override val twhinEmbeddingsFeature: Feature.Tensor = TwhinEmbeddingsFeatures.TwhinAuthorFollowEmbeddingsFeature override val commonFeatures: Set[Feature[_]] = Set.empty } object TwhinUserEngagementEmbeddingsAdapter extends TwhinEmbeddingsAdapter { override val twhinEmbeddingsFeature: Feature.Tensor = TwhinEmbeddingsFeatures.TwhinUserEngagementEmbeddingsFeature override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) } object TwhinRebuildUserEngagementEmbeddingsAdapter extends TwhinEmbeddingsAdapter { override val twhinEmbeddingsFeature: Feature.Tensor = TwhinEmbeddingsFeatures.TwhinRebuildUserEngagementEmbeddingsFeature override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) } object TwhinUserFollowEmbeddingsAdapter extends TwhinEmbeddingsAdapter { override val twhinEmbeddingsFeature: Feature.Tensor = TwhinEmbeddingsFeatures.TwhinUserFollowEmbeddingsFeature override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) } object TwhinUserPositiveEmbeddingsAdapter extends TwhinEmbeddingsAdapter { override val twhinEmbeddingsFeature: Feature.Tensor = TwhinEmbeddingsFeatures.TwhinUserPositiveEmbeddingsFeature override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) } object TwhinRebuildUserPositiveEmbeddingsAdapter extends TwhinEmbeddingsAdapter { override val twhinEmbeddingsFeature: Feature.Tensor = TwhinEmbeddingsFeatures.TwhinRebuildUserPositiveEmbeddingsFeature override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) } object TwhinTweetEmbeddingsAdapter extends TwhinEmbeddingsAdapter { override val twhinEmbeddingsFeature: Feature.Tensor = TwhinEmbeddingsFeatures.TwhinTweetEmbeddingsFeature override val commonFeatures: Set[Feature[_]] = Set.empty } object TwhinRebuildTweetEmbeddingsAdapter extends TwhinEmbeddingsAdapter { override val twhinEmbeddingsFeature: Feature.Tensor = TwhinEmbeddingsFeatures.TwhinRebuildTweetEmbeddingsFeature override val commonFeatures: Set[Feature[_]] = Set.empty } object TwhinVideoEmbeddingsAdapter extends TwhinEmbeddingsAdapter { override val twhinEmbeddingsFeature: Feature.Tensor = TwhinEmbeddingsFeatures.TwhinVideoEmbeddingsFeature override val commonFeatures: Set[Feature[_]] = Set.empty } object TwhinUserNegativeEmbeddingsAdapter extends TwhinEmbeddingsAdapter { override val twhinEmbeddingsFeature: Feature.Tensor = TwhinEmbeddingsFeatures.TwhinUserNegativeEmbeddingsFeature override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates import com.twitter.ml.api.FeatureContext import com.twitter.product_mixer.component_library.feature_hydrator.candidate.offline_aggregates.BaseAggregateRootFeature import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType.AggregateType import com.twitter.timelines.data_processing.ml_util.aggregation_framework.TypedAggregateGroup import scala.jdk.CollectionConverters.asJavaIterableConverter // A helper class deriving aggregate feature info from the given configuration parameters. class AggregateFeatureInfo( val aggregateGroups: Set[AggregateGroup], val aggregateType: AggregateType) { private val typedAggregateGroups = aggregateGroups.flatMap(_.buildTypedAggregateGroups()).toList val featureContext: FeatureContext = new FeatureContext( (typedAggregateGroups.flatMap(_.allOutputFeatures) ++ typedAggregateGroups.flatMap(_.allOutputKeys) ++ Seq(TypedAggregateGroup.timestampFeature)).asJava) val feature: BaseAggregateRootFeature = AggregateFeatureInfo.pickFeature(aggregateType) } object AggregateFeatureInfo { val features: Set[BaseAggregateRootFeature] = Set(PartAAggregateRootFeature, PartBAggregateRootFeature) def pickFeature(aggregateType: AggregateType): BaseAggregateRootFeature = { val filtered = features.filter(_.aggregateTypes.contains(aggregateType)) require( filtered.size == 1, "requested AggregateType must be backed by exactly one physical store.") filtered.head } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/offline_aggregates", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/offline_aggregates", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "servo/repo/src/main/scala", "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/timelines/prediction/adapters/request_context", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/scala/com/twitter/timelines/prediction/common/aggregates", "src/thrift/com/twitter/user_session_store:thrift-java", "timelines/data_processing/jobs/timeline_ranking_user_features:mini", "timelines/data_processing/ml_util/aggregation_framework:common_types", "timelines/data_processing/ml_util/aggregation_framework/conversion:for-timelines", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates import com.twitter.home_mixer.functional_component.feature_hydrator.WithDefaultFeatureMap import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.IRecordOneToOneAdapter import com.twitter.product_mixer.component_library.feature_hydrator.candidate.offline_aggregates.AggregateFeaturesToDecodeWithMetadata import com.twitter.product_mixer.component_library.feature_hydrator.candidate.offline_aggregates.BaseAggregateRootFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType.AggregateType import com.twitter.timelines.suggests.common.dense_data_record.thriftjava.DenseCompactDataRecord import com.twitter.util.Future import java.lang.{Long => JLong} import java.util.{Map => JMap} abstract case class BaseEdgeAggregateFeature( name: String, aggregateGroups: Set[AggregateGroup], aggregateType: AggregateType, extractMapFn: AggregateFeaturesToDecodeWithMetadata => JMap[JLong, DenseCompactDataRecord], adapter: IRecordOneToOneAdapter[Seq[DataRecord]], getSecondaryKeysFn: CandidateWithFeatures[TweetCandidate] => Seq[Long]) extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord private val rootFeatureInfo = new AggregateFeatureInfo(aggregateGroups, aggregateType) val featureContext: FeatureContext = rootFeatureInfo.featureContext val rootFeature: BaseAggregateRootFeature = rootFeatureInfo.feature override def hashCode(): Int = name.hashCode } trait BaseEdgeAggregateFeatureHydrator extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with WithDefaultFeatureMap { def aggregateFeatures: Set[BaseEdgeAggregateFeature] override def features = aggregateFeatures.asInstanceOf[Set[Feature[_, _]]] private val batchSize = 32 override lazy val defaultFeatureMap: FeatureMap = { val featureMapBuilder = new FeatureMapBuilder() aggregateFeatures.foreach(feature => featureMapBuilder.add(feature, feature.defaultValue)) featureMapBuilder.build() } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val featuresSeq = aggregateFeatures.toSeq val dataRecordsFuture = featuresSeq.map { feature => hydrateAggregateFeature(query, candidates, feature) } Future.collect(dataRecordsFuture).map { dataRecordsPerFeature => val dataRecordsPerCandidate = dataRecordsPerFeature.transpose dataRecordsPerCandidate.map { dataRecords => assert(featuresSeq.size == dataRecords.size) val builder = FeatureMapBuilder(sizeHint = featuresSeq.size) featuresSeq.zip(dataRecords).map { case (feature, dataRecord) => builder.add(feature, dataRecord) } builder.build() } } } private def getDataRecord( ids: Seq[Long], decoded: Map[Long, DataRecord], feature: BaseEdgeAggregateFeature ) = { val dataRecords = ids.flatMap(decoded.get) feature.adapter.adaptToDataRecord(dataRecords) } private def hydrateAggregateFeature( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]], feature: BaseEdgeAggregateFeature ): Future[Seq[DataRecord]] = { val rootFeature = feature.rootFeature val extractMapFn = feature.extractMapFn val featureContext = feature.featureContext val secondaryIds: Seq[Seq[Long]] = candidates.map(feature.getSecondaryKeysFn) val featuresToDecodeWithMetadata = query.features .flatMap(_.getOrElse(rootFeature, None)) .getOrElse(AggregateFeaturesToDecodeWithMetadata.empty) // Decode the DenseCompactDataRecords into DataRecords for each required secondary id. val decoded: Map[Long, DataRecord] = Utils.selectAndTransform( secondaryIds.flatten.distinct, featuresToDecodeWithMetadata.toDataRecord, extractMapFn(featuresToDecodeWithMetadata) ) // Remove unnecessary features in-place. This is safe because the underlying DataRecords // are unique and have just been generated in the previous step. decoded.values.foreach(Utils.filterDataRecord(_, featureContext)) // Put features into the FeatureMapBuilders OffloadFuturePools.offloadBatchElementToElement( secondaryIds, getDataRecord(_, decoded, feature), batchSize) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates import com.twitter.home_mixer.functional_component.feature_hydrator.TSPInferredTopicFeature import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.offline_aggregates.PassThroughAdapter import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.offline_aggregates.SparseAggregatesToDenseAdapter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig.CombineCountPolicies object EdgeAggregateFeatures { object UserAuthorAggregateFeature extends BaseEdgeAggregateFeature( name = "UserAuthorAggregateFeature", aggregateGroups = TimelinesAggregationConfig.userAuthorAggregatesV2 ++ Set( TimelinesAggregationConfig.userAuthorAggregatesV5, TimelinesAggregationConfig.tweetSourceUserAuthorAggregatesV1, TimelinesAggregationConfig.twitterWideUserAuthorAggregates ), aggregateType = AggregateType.UserAuthor, extractMapFn = _.userAuthorAggregates, adapter = PassThroughAdapter, getSecondaryKeysFn = _.features.getOrElse(AuthorIdFeature, None).toSeq ) object UserOriginalAuthorAggregateFeature extends BaseEdgeAggregateFeature( name = "UserOriginalAuthorAggregateFeature", aggregateGroups = Set(TimelinesAggregationConfig.userOriginalAuthorAggregatesV1), aggregateType = AggregateType.UserOriginalAuthor, extractMapFn = _.userOriginalAuthorAggregates, adapter = PassThroughAdapter, getSecondaryKeysFn = candidate => CandidatesUtil.getOriginalAuthorId(candidate.features).toSeq ) object UserTopicAggregateFeature extends BaseEdgeAggregateFeature( name = "UserTopicAggregateFeature", aggregateGroups = Set( TimelinesAggregationConfig.userTopicAggregates, TimelinesAggregationConfig.userTopicAggregatesV2, ), aggregateType = AggregateType.UserTopic, extractMapFn = _.userTopicAggregates, adapter = PassThroughAdapter, getSecondaryKeysFn = candidate => candidate.features.getOrElse(TopicIdSocialContextFeature, None).toSeq ) object UserMentionAggregateFeature extends BaseEdgeAggregateFeature( name = "UserMentionAggregateFeature", aggregateGroups = Set(TimelinesAggregationConfig.userMentionAggregates), aggregateType = AggregateType.UserMention, extractMapFn = _.userMentionAggregates, adapter = new SparseAggregatesToDenseAdapter(CombineCountPolicies.MentionCountsPolicy), getSecondaryKeysFn = candidate => candidate.features.getOrElse(MentionScreenNameFeature, Seq.empty).map(_.hashCode.toLong) ) object UserInferredTopicAggregateFeature extends BaseEdgeAggregateFeature( name = "UserInferredTopicAggregateFeature", aggregateGroups = Set( TimelinesAggregationConfig.userInferredTopicAggregates, ), aggregateType = AggregateType.UserInferredTopic, extractMapFn = _.userInferredTopicAggregates, adapter = new SparseAggregatesToDenseAdapter( CombineCountPolicies.UserInferredTopicCountsPolicy), getSecondaryKeysFn = candidate => candidate.features.getOrElse(TSPInferredTopicFeature, Map.empty[Long, Double]).keys.toSeq ) object UserInferredTopicAggregateV2Feature extends BaseEdgeAggregateFeature( name = "UserInferredTopicAggregateV2Feature", aggregateGroups = Set( TimelinesAggregationConfig.userInferredTopicAggregatesV2 ), aggregateType = AggregateType.UserInferredTopic, extractMapFn = _.userInferredTopicAggregates, adapter = new SparseAggregatesToDenseAdapter( CombineCountPolicies.UserInferredTopicV2CountsPolicy), getSecondaryKeysFn = candidate => candidate.features.getOrElse(TSPInferredTopicFeature, Map.empty[Long, Double]).keys.toSeq ) object UserMediaUnderstandingAnnotationAggregateFeature extends BaseEdgeAggregateFeature( name = "UserMediaUnderstandingAnnotationAggregateFeature", aggregateGroups = Set( TimelinesAggregationConfig.userMediaUnderstandingAnnotationAggregates), aggregateType = AggregateType.UserMediaUnderstandingAnnotation, extractMapFn = _.userMediaUnderstandingAnnotationAggregates, adapter = new SparseAggregatesToDenseAdapter( CombineCountPolicies.UserMediaUnderstandingAnnotationCountsPolicy), getSecondaryKeysFn = candidate => CandidatesUtil.getMediaUnderstandingAnnotationIds(candidate.features) ) object UserEngagerAggregateFeature extends BaseEdgeAggregateFeature( name = "UserEngagerAggregateFeature", aggregateGroups = Set(TimelinesAggregationConfig.userEngagerAggregates), aggregateType = AggregateType.UserEngager, extractMapFn = _.userEngagerAggregates, adapter = new SparseAggregatesToDenseAdapter(CombineCountPolicies.EngagerCountsPolicy), getSecondaryKeysFn = candidate => CandidatesUtil.getEngagerUserIds(candidate.features) ) object UserEngagerGoodClickAggregateFeature extends BaseEdgeAggregateFeature( name = "UserEngagerGoodClickAggregateFeature", aggregateGroups = Set(TimelinesAggregationConfig.userEngagerGoodClickAggregates), aggregateType = AggregateType.UserEngager, extractMapFn = _.userEngagerAggregates, adapter = new SparseAggregatesToDenseAdapter( CombineCountPolicies.EngagerGoodClickCountsPolicy), getSecondaryKeysFn = candidate => CandidatesUtil.getEngagerUserIds(candidate.features) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregateMetadataRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregatePartARepository import com.twitter.product_mixer.component_library.feature_hydrator.candidate.offline_aggregates.BaseAggregateQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.candidate.offline_aggregates.BaseAggregateRootFeature import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.servo.repository.Repository import com.twitter.timelines.data_processing.jobs.timeline_ranking_user_features.TimelinesPartAStoreRegister import com.twitter.timelines.data_processing.ml_util.aggregation_framework.StoreConfig import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata import com.twitter.user_session_store.thriftjava.UserSession import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton object PartAAggregateRootFeature extends BaseAggregateRootFeature { override val aggregateStores: Set[StoreConfig[_]] = TimelinesPartAStoreRegister.allStores } @Singleton class PartAAggregateQueryFeatureHydrator @Inject() ( @Named(TimelineAggregatePartARepository) repository: Repository[Long, Option[UserSession]], @Named(TimelineAggregateMetadataRepository) metadataRepository: Repository[Int, Option[DenseFeatureMetadata]]) extends BaseAggregateQueryFeatureHydrator( repository, metadataRepository, PartAAggregateRootFeature ) { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PartAAggregateQuery") override val features = Set(PartAAggregateRootFeature) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregateMetadataRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregatePartBRepository import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecordMerger import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.product_mixer.component_library.feature_hydrator.candidate.offline_aggregates.BaseAggregateQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.candidate.offline_aggregates.BaseAggregateRootFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.repository.Repository import com.twitter.stitch.Stitch import com.twitter.timelines.data_processing.jobs.timeline_ranking_user_features.TimelinesPartBStoreRegister import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType import com.twitter.timelines.data_processing.ml_util.aggregation_framework.StoreConfig import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata import com.twitter.user_session_store.thriftjava.UserSession import com.twitter.util.Time import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton object PartBAggregateRootFeature extends BaseAggregateRootFeature { override val aggregateStores: Set[StoreConfig[_]] = TimelinesPartBStoreRegister.allStores } object UserAggregateFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class PartBAggregateQueryFeatureHydrator @Inject() ( @Named(TimelineAggregatePartBRepository) repository: Repository[Long, Option[UserSession]], @Named(TimelineAggregateMetadataRepository) metadataRepository: Repository[Int, Option[DenseFeatureMetadata]]) extends BaseAggregateQueryFeatureHydrator( repository, metadataRepository, PartBAggregateRootFeature ) { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PartBAggregateQuery") override val features: Set[Feature[_, _]] = Set(PartBAggregateRootFeature, UserAggregateFeature) private val userAggregateFeatureInfo = new AggregateFeatureInfo( aggregateGroups = Set( TimelinesAggregationConfig.userAggregatesV2, TimelinesAggregationConfig.userAggregatesV5Continuous, TimelinesAggregationConfig.userAggregatesV6, TimelinesAggregationConfig.twitterWideUserAggregates, ), aggregateType = AggregateType.User ) private val userHourAggregateFeatureInfo = new AggregateFeatureInfo( aggregateGroups = Set( TimelinesAggregationConfig.userRequestHourAggregates, ), aggregateType = AggregateType.UserRequestHour ) private val userDowAggregateFeatureInfo = new AggregateFeatureInfo( aggregateGroups = Set( TimelinesAggregationConfig.userRequestDowAggregates ), aggregateType = AggregateType.UserRequestDow ) require( userAggregateFeatureInfo.feature == PartBAggregateRootFeature, "UserAggregates feature must be provided by the PartB data source.") require( userHourAggregateFeatureInfo.feature == PartBAggregateRootFeature, "UserRequstHourAggregates feature must be provided by the PartB data source.") require( userDowAggregateFeatureInfo.feature == PartBAggregateRootFeature, "UserRequestDowAggregates feature must be provided by the PartB data source.") override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { // Hydrate TimelineAggregatePartBFeature and UserAggregateFeature sequentially. super.hydrate(query).map { featureMap => val time: Time = Time.now val hourOfDay = RequestContextAdapter.hourFromTimestamp(time.inMilliseconds) val dayOfWeek = RequestContextAdapter.dowFromTimestamp(time.inMilliseconds) val dr = featureMap .get(PartBAggregateRootFeature).map { featuresWithMetadata => val userAggregatesDr = featuresWithMetadata.userAggregatesOpt .map(featuresWithMetadata.toDataRecord) val userRequestHourAggregatesDr = Option(featuresWithMetadata.userRequestHourAggregates.get(hourOfDay)) .map(featuresWithMetadata.toDataRecord) val userRequestDowAggregatesDr = Option(featuresWithMetadata.userRequestDowAggregates.get(dayOfWeek)) .map(featuresWithMetadata.toDataRecord) dropUnknownFeatures(userAggregatesDr, userAggregateFeatureInfo.featureContext) dropUnknownFeatures( userRequestHourAggregatesDr, userHourAggregateFeatureInfo.featureContext) dropUnknownFeatures( userRequestDowAggregatesDr, userDowAggregateFeatureInfo.featureContext) mergeDataRecordOpts( userAggregatesDr, userRequestHourAggregatesDr, userRequestDowAggregatesDr) }.getOrElse(new DataRecord()) featureMap + (UserAggregateFeature, dr) } } private val drMerger = new DataRecordMerger private def mergeDataRecordOpts(dataRecordOpts: Option[DataRecord]*): DataRecord = dataRecordOpts.flatten.foldLeft(new DataRecord) { (l, r) => drMerger.merge(l, r) l } private def dropUnknownFeatures( dataRecordOpt: Option[DataRecord], featureContext: FeatureContext ): Unit = dataRecordOpt.foreach(new RichDataRecord(_, featureContext).dropUnknownFeatures()) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/TopicEdgeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserInferredTopicAggregateFeature import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserInferredTopicAggregateV2Feature import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserTopicAggregateFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableTopicEdgeAggregateFeatureHydratorParam import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery object TopicEdgeAggregateFeatureHydrator extends BaseEdgeAggregateFeatureHydrator with Conditionally[PipelineQuery] { override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableTopicEdgeAggregateFeatureHydratorParam) override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TopicEdgeAggregate") override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = Set( UserInferredTopicAggregateFeature, UserInferredTopicAggregateV2Feature, UserTopicAggregateFeature, ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/TopicEdgeTruncatedAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserInferredTopicAggregateFeature import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserInferredTopicAggregateV2Feature import com.twitter.home_mixer.param.HomeGlobalParams.EnableTopicEdgeAggregateFeatureHydratorParam import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery object TopicEdgeTruncatedAggregateFeatureHydrator extends BaseEdgeAggregateFeatureHydrator with Conditionally[PipelineQuery] { override def onlyIf(query: PipelineQuery): Boolean = !query.params(EnableTopicEdgeAggregateFeatureHydratorParam) override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TopicEdgeTruncatedAggregate") override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = Set( UserInferredTopicAggregateFeature, UserInferredTopicAggregateV2Feature, ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/TweetContentEdgeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserMediaUnderstandingAnnotationAggregateFeature import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier object TweetContentEdgeAggregateFeatureHydrator extends BaseEdgeAggregateFeatureHydrator { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetContentEdgeAggregate") override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = Set( UserMediaUnderstandingAnnotationAggregateFeature ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/UserEngagerEdgeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserEngagerAggregateFeature import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserEngagerGoodClickAggregateFeature import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier object UserEngagerEdgeAggregateFeatureHydrator extends BaseEdgeAggregateFeatureHydrator { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserEngagerEdgeAggregate") override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = Set( UserEngagerAggregateFeature, UserEngagerGoodClickAggregateFeature, ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/UserEntityEdgeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserAuthorAggregateFeature import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserMentionAggregateFeature import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserOriginalAuthorAggregateFeature import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier object UserEntityAggregateFeatureHydrator extends BaseEdgeAggregateFeatureHydrator { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserEntityEdgeAggregate") override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = Set( UserAuthorAggregateFeature, UserOriginalAuthorAggregateFeature, UserMentionAggregateFeature ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Utils.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.timelines.suggests.common.dense_data_record.thriftjava.DenseCompactDataRecord private[offline_aggregates] object Utils { /** * Selects only those values in map that correspond to the keys in ids and apply the provided * transform to the selected values. This is a convenience method for use by Timelines Aggregation * Framework based features. * * @param idsToSelect The set of ids to extract values for. * @param transform A transform to apply to the selected values. * @param map Map[Long, DenseCompactDataRecord] */ def selectAndTransform( idsToSelect: Seq[Long], transform: DenseCompactDataRecord => DataRecord, map: java.util.Map[java.lang.Long, DenseCompactDataRecord], ): Map[Long, DataRecord] = { val filtered: Seq[(Long, DataRecord)] = for { id <- idsToSelect if map.containsKey(id) } yield { id -> transform(map.get(id)) } filtered.toMap } def filterDataRecord(dr: DataRecord, featureContext: FeatureContext): Unit = { new RichDataRecord(dr, featureContext).dropUnknownFeatures() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/src/jvm/com/twitter/storehaus:core", "finatra/inject/inject-core/src/main/scala", "home-mixer-features/thrift/src/main/thrift:thrift-java", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/real_time_aggregates", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/real_time_aggregates", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "servo/repo/src/main/scala", "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/ml/api", "src/scala/com/twitter/ml/api:api-base", "src/scala/com/twitter/storehaus_internal/store", "src/scala/com/twitter/timelines/prediction/common/aggregates/real_time:base-config", "src/scala/com/twitter/timelines/prediction/common/aggregates/real_time/tv:base-config", "src/thrift/com/twitter/timelines/realtime_aggregates:thrift-scala", ], exports = [ "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/real_time_aggregates", "timelines/data_processing/ml_util/aggregation_framework/real_time_aggregates", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.authorIdFeature import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.keyTransformD1 import com.twitter.home_mixer.param.HomeMixerInjectionNames.EngagementsReceivedByAuthorCache import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer_features.{thriftjava => t} import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.ReadCache import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import javax.inject.Inject import javax.inject.Singleton object EngagementsReceivedByAuthorRealTimeAggregateFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator @Inject() ( override val homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient, @Named(EngagementsReceivedByAuthorCache) override val client: ReadCache[Long, DataRecord], override val statsReceiver: StatsReceiver) extends FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator[Long] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("EngagementsReceivedByAuthorRealTimeAggregate") override val outputFeature: DataRecordInAFeature[TweetCandidate] = EngagementsReceivedByAuthorRealTimeAggregateFeature override val aggregateGroups: Seq[AggregateGroup] = Seq( authorEngagementRealTimeAggregatesProd, authorShareEngagementsRealTimeAggregates ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( authorShareEngagementsRealTimeAggregates -> "original_author.timelines.author_share_engagements_real_time_aggregates." ) def serializeKey(key: Long): String = { keyTransformD1(authorIdFeature)(key) } override def keysFromQueryAndCandidates( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[Long]] = candidates.map(candidate => CandidatesUtil.getOriginalAuthorId(candidate.features)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates import com.twitter.home_mixer.functional_component.feature_hydrator.WithDefaultFeatureMap import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableHomeMixerFeaturesService import com.twitter.home_mixer.util.MissingKeyException import com.twitter.home_mixer_features.thriftjava.HomeMixerFeaturesRequest import com.twitter.home_mixer_features.{thriftjava => t} import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.feature_hydrator.candidate.real_time_aggregates.BaseRealTimeAggregateBulkCandidateFeatureHydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.util.Future import com.twitter.util.Return import com.twitter.util.Throw import com.twitter.util.Try import scala.jdk.CollectionConverters.iterableAsScalaIterableConverter import scala.jdk.CollectionConverters.seqAsJavaListConverter trait FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator[K] extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[K, TweetCandidate] with WithDefaultFeatureMap { def EmptyDataRecord: DataRecord = new DataRecord() override lazy val defaultFeatureMap: FeatureMap = FeatureMap(outputFeature, EmptyDataRecord) def serializeKey(key: K): String val homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient private val batchSize = 64 private def createHomeMixerFeaturesRequest(keys: Seq[K]): t.HomeMixerFeaturesRequest = { val keysSerialized = keys.map(serializeKey) val request = new HomeMixerFeaturesRequest() request.setKeys(keysSerialized.asJava) request.setCache(t.Cache.RTA) } private def unmarshallHomeMixerFeaturesResponse( response: t.HomeMixerFeaturesResponse ): Iterable[DataRecord] = { response.getHomeMixerFeatures.asScala .map { homeMixerFeatureOpt => if (homeMixerFeatureOpt.isSetHomeMixerFeaturesType) { val homeMixerFeature = homeMixerFeatureOpt.getHomeMixerFeaturesType if (homeMixerFeature.isSet(t.HomeMixerFeaturesType._Fields.DATA_RECORD)) { postTransformer(homeMixerFeature.getDataRecord) } else { throw new Exception("Unexpected type") } } else EmptyDataRecord } } private def getFeatureMaps( possiblyKeys: Seq[Option[K]], dataRecordMap: Map[K, DataRecord] ): Future[Seq[Try[DataRecord]]] = { val transformer = { key: Option[K] => if (key.nonEmpty) Return(dataRecordMap.getOrElse(key.get, EmptyDataRecord)) else Throw(MissingKeyException) } OffloadFuturePools.offloadBatchElementToElement(possiblyKeys, transformer, batchSize) } def fetchRecordsFromHomeMixerFeaturesService( possiblyKeys: Seq[Option[K]] ): Future[Seq[Try[DataRecord]]] = { val keys = possiblyKeys.flatten.distinct val transformer = { keyGroup: Seq[K] => val request = createHomeMixerFeaturesRequest(keyGroup) val responseFut = homeMixerFeatureService.getHomeMixerFeatures(request) responseFut .map { response => keyGroup.zip(unmarshallHomeMixerFeaturesResponse(response)) }.handle { case _ => Seq.empty } } val response = OffloadFuturePools.offloadBatchSeqToFutureSeq(keys, transformer, batchSize) response.map(_.toMap).flatMap { keyValueResult => getFeatureMaps(possiblyKeys, keyValueResult) } } def fetchRecords( possiblyKeys: Seq[Option[K]], callMiddleMan: Boolean ): Future[Seq[Try[DataRecord]]] = { if (callMiddleMan) fetchRecordsFromHomeMixerFeaturesService(possiblyKeys) else fetchAndConstructDataRecords(possiblyKeys) // cache is default } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val possiblyKeys = keysFromQueryAndCandidates(query, candidates) val callMiddleMan = query.params(EnableHomeMixerFeaturesService) fetchRecords(possiblyKeys, callMiddleMan).map { dataRecords => val featureMaps = dataRecords.map { dataRecord => FeatureMapBuilder().add(outputFeature, dataRecord).build() } featureMaps } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.countryCodeFeature import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.keyTransformD1T1 import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.topicIdFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableTopicCountryBasedRealTimeAggregateFeatureHydratorParam import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicCountryEngagementCache import com.twitter.home_mixer_features.{thriftjava => t} import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.ReadCache import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import javax.inject.Inject import javax.inject.Singleton object TopicCountryEngagementRealTimeAggregateFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TopicCountryEngagementRealTimeAggregateFeatureHydrator @Inject() ( override val homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient, @Named(TopicCountryEngagementCache) override val client: ReadCache[(Long, String), DataRecord], override val statsReceiver: StatsReceiver) extends FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator[(Long, String)] with Conditionally[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TopicCountryEngagementRealTimeAggregate") override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableTopicCountryBasedRealTimeAggregateFeatureHydratorParam) override val outputFeature: DataRecordInAFeature[TweetCandidate] = TopicCountryEngagementRealTimeAggregateFeature override val aggregateGroups: Seq[AggregateGroup] = Seq( topicCountryRealTimeAggregates ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( topicCountryRealTimeAggregates -> "topic-country_code.timelines.topic_country_engagement_real_time_aggregates." ) def serializeKey(key: (Long, String)): String = { keyTransformD1T1(topicIdFeature, countryCodeFeature)(key) } override def keysFromQueryAndCandidates( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[(Long, String)]] = { candidates.map { candidate => val maybeTopicId = candidate.features .getTry(TopicIdSocialContextFeature) .toOption .flatten val maybeCountryCode = query.clientContext.countryCode for { topicId <- maybeTopicId countryCode <- maybeCountryCode } yield (topicId, countryCode) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.keyTransformD1 import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.topicIdFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableTopicBasedRealTimeAggregateFeatureHydratorParam import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicEngagementCache import com.twitter.home_mixer_features.{thriftjava => t} import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.ReadCache import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton object TopicEngagementRealTimeAggregateFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TopicEngagementRealTimeAggregateFeatureHydrator @Inject() ( override val homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient, @Named(TopicEngagementCache) override val client: ReadCache[Long, DataRecord], override val statsReceiver: StatsReceiver) extends FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator[Long] with Conditionally[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TopicEngagementRealTimeAggregate") override val outputFeature: DataRecordInAFeature[TweetCandidate] = TopicEngagementRealTimeAggregateFeature override val aggregateGroups: Seq[AggregateGroup] = Seq( topicEngagementRealTimeAggregatesProd, topicEngagement24HourRealTimeAggregatesProd, topicShareEngagementsRealTimeAggregates ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( topicEngagement24HourRealTimeAggregatesProd -> "topic.timelines.topic_engagement_24_hour_real_time_aggregates.", topicShareEngagementsRealTimeAggregates -> "topic.timelines.topic_share_engagements_real_time_aggregates." ) override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableTopicBasedRealTimeAggregateFeatureHydratorParam) def serializeKey(key: Long): String = { keyTransformD1(topicIdFeature)(key) } override def keysFromQueryAndCandidates( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[Long]] = { candidates.map { candidate => candidate.features .getTry(TopicIdSocialContextFeature) .toOption .flatten } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.countryCodeFeature import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.keyTransformD1T1 import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.tweetIdFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableTweetCountryRTAMhFallbackParam import com.twitter.home_mixer.param.HomeGlobalParams.EnableTweetCountryRTAMhOnlyParam import com.twitter.home_mixer.param.HomeMixerInjectionNames.RTAManhattanStore import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetCountryEngagementCache import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer_features.{thriftjava => t} import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.cache.ReadCache import com.twitter.storehaus.ReadableStore import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregationKey import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import com.twitter.util.Future import com.twitter.util.Try import com.twitter.timelines.realtime_aggregates.{thriftscala => thrift} import javax.inject.Inject import javax.inject.Singleton import com.twitter.stitch.Stitch object TweetCountryEngagementRealTimeAggregateFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TweetCountryEngagementRealTimeAggregateFeatureHydrator @Inject() ( override val homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient, @Named(TweetCountryEngagementCache) override val client: ReadCache[(Long, String), DataRecord], @Named(RTAManhattanStore) mhClient: Option[ReadableStore[thrift.AggregationKey, DataRecord]], override val statsReceiver: StatsReceiver) extends FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator[(Long, String)] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetCountryEngagementRealTimeAggregate") override val outputFeature: DataRecordInAFeature[TweetCandidate] = TweetCountryEngagementRealTimeAggregateFeature override val aggregateGroups: Seq[AggregateGroup] = Seq( tweetCountryRealTimeAggregates, tweetCountryPrivateEngagementsRealTimeAggregates ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( tweetCountryRealTimeAggregates -> "tweet-country_code.timelines.tweet_country_engagement_real_time_aggregates.", tweetCountryPrivateEngagementsRealTimeAggregates -> "tweet-country_code.timelines.tweet_country_private_engagement_real_time_aggregates." ) def serializeKey(key: (Long, String)): String = { keyTransformD1T1(tweetIdFeature, countryCodeFeature)(key) } override def keysFromQueryAndCandidates( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[(Long, String)]] = { val countryCode = query.clientContext.countryCode candidates.map { candidate => val originalTweetId = CandidatesUtil.getOriginalTweetId(candidate) countryCode.map((originalTweetId, _)) } } def convert(key: (Long, String)): thrift.AggregationKey = { val ak = AggregationKey(Map(tweetIdFeature -> key._1), Map(countryCodeFeature -> key._2)) thrift.AggregationKey( ak.discreteFeaturesById, ak.textFeaturesById ) } def fetchAndConstructDataRecordFromMh( possiblyKeys: Seq[Option[(Long, String)]] ): Future[Seq[Try[DataRecord]]] = { Future .collect { possiblyKeys.flatten .map { convert(_) } .grouped(64).map { keyGroup => val results = mhClient.get.multiGet(keyGroup.toSet) Future.collect(keyGroup.flatMap(results.get)).map { drSeq => drSeq.map { drOpt => if (drOpt.isEmpty) statsReceiver.scope("mhTweetCountryRTA").counter("empty").incr() else statsReceiver.scope("mhTweetCountryRTA").counter("non_empty").incr() Try(drOpt.map(postTransformer).getOrElse(EmptyDataRecord)) } } }.toSeq }.map(_.flatten) } def getFetchFunc( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Future[Seq[Try[DataRecord]]] = { val fetchFromMhOnly = query.params.getBoolean(EnableTweetCountryRTAMhOnlyParam) val fetchFromMhAsFallBack = query.params.getBoolean(EnableTweetCountryRTAMhFallbackParam) val possiblyKeys = keysFromQueryAndCandidates(query, candidates) val stats = statsReceiver.scope("tweet_country_real_time_rta") if (fetchFromMhOnly) { fetchAndConstructDataRecordFromMh(possiblyKeys) } else if (fetchFromMhAsFallBack) { fetchAndConstructDataRecordsWithFallback( possiblyKeys, stats, fetchAndConstructDataRecords, fetchAndConstructDataRecordFromMh, ) } else { fetchAndConstructDataRecords(possiblyKeys) } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { getFetchFunc(query, candidates).map { dataRecords => val featureMaps = dataRecords.map { dataRecord => FeatureMapBuilder().add(outputFeature, dataRecord).build() } featureMaps } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TweetEngagementRealTimeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.keyTransformD1 import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.tweetIdFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableTweetRTAMhFallbackParam import com.twitter.home_mixer.param.HomeGlobalParams.EnableTweetRTAMhOnlyParam import com.twitter.home_mixer.param.HomeMixerInjectionNames.RTAManhattanStore import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetEngagementCache import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer_features.{thriftjava => t} import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.cache.ReadCache import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregationKey import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import com.twitter.util.Future import com.twitter.util.Try import com.twitter.storehaus.ReadableStore import com.twitter.timelines.realtime_aggregates.{thriftscala => thrift} import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton object TweetEngagementRealTimeAggregateFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TweetEngagementRealTimeAggregateFeatureHydrator @Inject() ( override val homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient, @Named(TweetEngagementCache) override val client: ReadCache[Long, DataRecord], @Named(RTAManhattanStore) mhClient: Option[ReadableStore[thrift.AggregationKey, DataRecord]], override val statsReceiver: StatsReceiver) extends FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator[Long] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetEngagementRealTimeAggregate") override val outputFeature: DataRecordInAFeature[TweetCandidate] = TweetEngagementRealTimeAggregateFeature val batchSize = 64 override val aggregateGroups: Seq[AggregateGroup] = Seq( tweetEngagement30MinuteCountsProd, tweetEngagementTotalCountsProd, tweetEngagementUserStateRealTimeAggregatesProd, tweetNegativeEngagementUserStateRealTimeAggregates, tweetNegativeEngagement6HourCounts, tweetNegativeEngagementTotalCounts, tweetShareEngagementsRealTimeAggregates, tweetBCEDwellEngagementsRealTimeAggregates ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( tweetShareEngagementsRealTimeAggregates -> "original_tweet.timelines.tweet_share_engagements_real_time_aggregates.", tweetBCEDwellEngagementsRealTimeAggregates -> "original_tweet.timelines.tweet_bce_dwell_engagements_real_time_aggregates." ) def serializeKey(key: Long): String = { keyTransformD1(tweetIdFeature)(key) } override def keysFromQueryAndCandidates( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[Long]] = { val keys = candidates .map(candidate => Some(CandidatesUtil.getOriginalTweetId(candidate))) keys } def convert(key: Long): thrift.AggregationKey = { val ak = AggregationKey(Map(tweetIdFeature -> key), Map.empty) thrift.AggregationKey( ak.discreteFeaturesById, ak.textFeaturesById ) } def fetchAndConstructDataRecordFromMh( possiblyKeys: Seq[Option[Long]] ): Future[Seq[Try[DataRecord]]] = { Future .collect { possiblyKeys.flatten .map { convert(_) } .grouped(batchSize).map { keyGroup => val results = mhClient.get.multiGet(keyGroup.toSet) Future.collect(keyGroup.flatMap(results.get)).map { drSeq => drSeq.map { drOpt => if (drOpt.isEmpty) statsReceiver.scope("mhTweetRTA").counter("empty").incr() else statsReceiver.scope("mhTweetRTA").counter("non_empty").incr() Try(drOpt.map(postTransformer).getOrElse(EmptyDataRecord)) } } }.toSeq }.map(_.flatten) } def getFetchFunc( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Future[Seq[Try[DataRecord]]] = { val fetchFromMhOnly = query.params.getBoolean(EnableTweetRTAMhOnlyParam) val fetchFromMhAsFallBack = query.params.getBoolean(EnableTweetRTAMhFallbackParam) val possiblyKeys = keysFromQueryAndCandidates(query, candidates) val stats = statsReceiver.scope("tweet_real_time_rta") if (fetchFromMhOnly) { fetchAndConstructDataRecordFromMh(possiblyKeys) } else if (fetchFromMhAsFallBack) { fetchAndConstructDataRecordsWithFallback( possiblyKeys, stats, fetchAndConstructDataRecords, fetchAndConstructDataRecordFromMh, ) } else { fetchAndConstructDataRecords(possiblyKeys) } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { getFetchFunc(query, candidates).map { dataRecords => val featureMaps = dataRecords.map { dataRecord => FeatureMapBuilder().add(outputFeature, dataRecord).build() } featureMaps } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/TwitterListEngagementRealTimeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.ListIdFeature import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.keyTransformD1 import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.listIdFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwitterListEngagementCache import com.twitter.home_mixer_features.{thriftjava => t} import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.ReadCache import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import javax.inject.Inject import javax.inject.Singleton object TwitterListEngagementRealTimeAggregateFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TwitterListEngagementRealTimeAggregateFeatureHydrator @Inject() ( override val homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient, @Named(TwitterListEngagementCache) override val client: ReadCache[Long, DataRecord], override val statsReceiver: StatsReceiver) extends FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator[Long] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TwitterListEngagementRealTimeAggregate") override val outputFeature: DataRecordInAFeature[TweetCandidate] = TwitterListEngagementRealTimeAggregateFeature override val aggregateGroups: Seq[AggregateGroup] = Seq(listEngagementRealTimeAggregatesProd) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( listEngagementRealTimeAggregatesProd -> "twitter_list.timelines.twitter_list_engagement_real_time_aggregates." ) def serializeKey(key: Long): String = { keyTransformD1(listIdFeature)(key) } override def keysFromQueryAndCandidates( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[Long]] = candidates.map { candidate => candidate.features.getTry(ListIdFeature).toOption.flatten } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.authorIdFeature import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.keyTransformD2 import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.keyTransformD2AggregationKey import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.userIdFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableUserAuthorRTAMhFallbackParam import com.twitter.home_mixer.param.HomeGlobalParams.EnableUserAuthorRTAMhOnlyParam import com.twitter.home_mixer.param.HomeMixerInjectionNames.RTAManhattanStore import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserAuthorEngagementCache import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer_features.{thriftjava => t} import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.cache.ReadCache import com.twitter.stitch.Stitch import com.twitter.storehaus.ReadableStore import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import com.twitter.timelines.realtime_aggregates.{thriftscala => thrift} import com.twitter.util.Future import com.twitter.util.Try import javax.inject.Inject import javax.inject.Singleton object UserAuthorEngagementRealTimeAggregateFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class UserAuthorEngagementRealTimeAggregateFeatureHydrator @Inject() ( override val homeMixerFeatureService: t.HomeMixerFeatures.ServiceToClient, @Named(UserAuthorEngagementCache) override val client: ReadCache[(Long, Long), DataRecord], @Named(RTAManhattanStore) mhClient: Option[ ReadableStore[thrift.AggregationKey, DataRecord] ], override val statsReceiver: StatsReceiver) extends FlagBasedRealTimeAggregateBulkCandidateFeatureHydrator[(Long, Long)] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserAuthorEngagementRealTimeAggregate") override val outputFeature: DataRecordInAFeature[TweetCandidate] = UserAuthorEngagementRealTimeAggregateFeature override val aggregateGroups: Seq[AggregateGroup] = Seq( userAuthorEngagementRealTimeAggregatesProd, userAuthorShareEngagementsRealTimeAggregates ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( userAuthorEngagementRealTimeAggregatesProd -> "user-author.timelines.user_author_engagement_real_time_aggregates.", userAuthorShareEngagementsRealTimeAggregates -> "user-author.timelines.user_author_share_engagements_real_time_aggregates." ) def serializeKey(key: (Long, Long)): String = { keyTransformD2(userIdFeature, authorIdFeature)(key) } override def keysFromQueryAndCandidates( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[(Long, Long)]] = { val userId = query.getRequiredUserId candidates.map { candidate => CandidatesUtil .getOriginalAuthorId(candidate.features) .map((userId, _)) } } def convert(key: (Long, Long)): thrift.AggregationKey = { val ak = keyTransformD2AggregationKey(userIdFeature, authorIdFeature)((key._1, key._2)) thrift.AggregationKey( ak.discreteFeaturesById, ak.textFeaturesById ) } def fetchAndConstructDataRecordFromMh( possiblyKeys: Seq[Option[(Long, Long)]] ): Future[Seq[Try[DataRecord]]] = { Future .collect { possiblyKeys.flatten .map { convert(_) } .grouped(64).map { keyGroup => val results = mhClient.get.multiGet(keyGroup.toSet) Future.collect(keyGroup.flatMap(results.get)).map { drSeq => drSeq.map { drOpt => if (drOpt.isEmpty) statsReceiver.scope("mhUserAuthorRTA").counter("empty").incr() else statsReceiver.scope("mhUserAuthorRTA").counter("non_empty").incr() Try(drOpt.map(postTransformer).getOrElse(EmptyDataRecord)) } } }.toSeq }.map(_.flatten) } def getFetchFunc( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Future[Seq[Try[DataRecord]]] = { val fetchFromMhOnly = query.params.getBoolean(EnableUserAuthorRTAMhOnlyParam) val fetchFromMhAsFallBack = query.params.getBoolean(EnableUserAuthorRTAMhFallbackParam) val possiblyKeys = keysFromQueryAndCandidates(query, candidates) val stats = statsReceiver.scope("user_author_real_time_rta") if (fetchFromMhOnly) { fetchAndConstructDataRecordFromMh(possiblyKeys) } else if (fetchFromMhAsFallBack) { fetchAndConstructDataRecordsWithFallback( possiblyKeys, stats, fetchAndConstructDataRecords, fetchAndConstructDataRecordFromMh, ) } else { fetchAndConstructDataRecords(possiblyKeys) } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { getFetchFunc(query, candidates).map { dataRecords => val featureMaps = dataRecords.map { dataRecord => FeatureMapBuilder().add(outputFeature, dataRecord).build() } featureMaps } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserEngagementRealTimeAggregatesFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.module.RealtimeAggregateFeatureRepositoryModule.userIdFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableUserRTAMhFallbackParam import com.twitter.home_mixer.param.HomeGlobalParams.EnableUserRTAMhOnlyParam import com.twitter.home_mixer.param.HomeMixerInjectionNames.RTAManhattanStore import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserEngagementCache import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.feature_hydrator.query.real_time_aggregates.BaseRealTimeAggregateQueryFeatureHydrator import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.cache.ReadCache import com.twitter.stitch.Stitch import com.twitter.storehaus.ReadableStore import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregationKey import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import com.twitter.util.Future import com.twitter.util.Try import com.twitter.timelines.realtime_aggregates.{thriftscala => thrift} import javax.inject.Inject import javax.inject.Singleton object UserEngagementRealTimeAggregateFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class UserEngagementRealTimeAggregatesFeatureHydrator @Inject() ( @Named(UserEngagementCache) override val client: ReadCache[Long, DataRecord], @Named(RTAManhattanStore) mhClient: Option[ReadableStore[thrift.AggregationKey, DataRecord]], override val statsReceiver: StatsReceiver) extends BaseRealTimeAggregateQueryFeatureHydrator[Long] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserEngagementRealTimeAggregates") override val outputFeature: DataRecordInAFeature[PipelineQuery] = UserEngagementRealTimeAggregateFeature val aggregateGroups: Seq[AggregateGroup] = Seq( userEngagementRealTimeAggregatesProd, userShareEngagementsRealTimeAggregates, userBCEDwellEngagementsRealTimeAggregates, userEngagement48HourRealTimeAggregatesProd, userNegativeEngagementAuthorUserState72HourRealTimeAggregates, userNegativeEngagementAuthorUserStateRealTimeAggregates, userProfileEngagementRealTimeAggregates, ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( userShareEngagementsRealTimeAggregates -> "user.timelines.user_share_engagements_real_time_aggregates.", userBCEDwellEngagementsRealTimeAggregates -> "user.timelines.user_bce_dwell_engagements_real_time_aggregates.", userEngagement48HourRealTimeAggregatesProd -> "user.timelines.user_engagement_48_hour_real_time_aggregates.", userNegativeEngagementAuthorUserState72HourRealTimeAggregates -> "user.timelines.user_negative_engagement_author_user_state_72_hour_real_time_aggregates.", userProfileEngagementRealTimeAggregates -> "user.timelines.user_profile_engagement_real_time_aggregates." ) override def keysFromQueryAndCandidates(query: PipelineQuery): Option[Long] = { Some(query.getRequiredUserId) } val EmptyDataRecord = new DataRecord override def fetchAndConstructDataRecords( possiblyKeys: Seq[Option[Long]] ): Future[Seq[Try[DataRecord]]] = { val keys = possiblyKeys.flatten.map { k => val ak = AggregationKey(Map(userIdFeature -> k), Map.empty) thrift.AggregationKey( ak.discreteFeaturesById, ak.textFeaturesById ) } Future .collect { keys.map { key => statsReceiver.scope("mhUserRTA").counter("mh_called").incr() val result = mhClient.get.get(key) result.map { drOpt => if (drOpt.isEmpty) statsReceiver.scope("mhUserRTA").counter("empty").incr() else statsReceiver.scope("mhUserRTA").counter("non_empty").incr() Try(drOpt.map(postTransformer).getOrElse(EmptyDataRecord)) } } } } def convert(key: Long): thrift.AggregationKey = { val ak = AggregationKey(Map(userIdFeature -> key), Map.empty) thrift.AggregationKey( ak.discreteFeaturesById, ak.textFeaturesById ) } def fetchAndConstructDataRecordFromMh( possiblyKeys: Seq[Option[Long]] ): Future[Seq[Try[DataRecord]]] = { Future .collect { possiblyKeys.flatten .map { convert(_) } .grouped(64).map { keyGroup => val results = mhClient.get.multiGet(keyGroup.toSet) Future.collect(keyGroup.flatMap(results.get)).map { drSeq => drSeq.map { drOpt => if (drOpt.isEmpty) statsReceiver.scope("mhUserRTA").counter("empty").incr() else statsReceiver.scope("mhUserRTA").counter("non_empty").incr() Try(drOpt.map(postTransformer).getOrElse(EmptyDataRecord)) } } }.toSeq }.map(_.flatten) } def getFetchFunc( query: PipelineQuery ): Future[Seq[Try[DataRecord]]] = { val fetchFromMhOnly = query.params.getBoolean(EnableUserRTAMhOnlyParam) val fetchFromMhAsFallBack = query.params.getBoolean(EnableUserRTAMhFallbackParam) val possiblyKeys = Seq(keysFromQueryAndCandidates(query)) val stats = statsReceiver.scope("user_author_real_time_rta") if (fetchFromMhOnly) { fetchAndConstructDataRecordFromMh(possiblyKeys) } else if (fetchFromMhAsFallBack) { fetchAndConstructDataRecordsWithFallback( possiblyKeys, stats, fetchAndConstructDataRecords, fetchAndConstructDataRecordFromMh, ) } else { fetchAndConstructDataRecords(possiblyKeys) } } override def hydrate( query: PipelineQuery ): Stitch[FeatureMap] = OffloadFuturePools.offloadFuture { getFetchFunc(query).map { dataRecords => val featureMaps = dataRecords.map { dataRecord => FeatureMapBuilder().add(outputFeature, dataRecord).build() } featureMaps.head } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates/UserTweetTvVideoRealTimeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.TvVideoByUserTweetCache import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.feature_hydrator.candidate.real_time_aggregates.BaseRealTimeAggregateBulkCandidateFeatureHydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.cache.ReadCache import com.twitter.stitch.Stitch import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.data_processing.ml_util.aggregation_framework.metrics.AggregateFeature import com.twitter.timelines.prediction.common.aggregates.real_time.tv.TvOnlineAggregationFeaturesOnlyConfig import com.twitter.util.Return import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton object DummyFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = { new DataRecord() } } // The feature value is keyed by the RTA aggregation group prefix, then the AggregateFeature // for ease of access. object UserTweetTvVideoRealtimeAggregatesFeatures extends Feature[TweetCandidate, Option[Map[String, Map[AggregateFeature[_], Double]]]] @Singleton class UserTweetTvVideoRealTimeAggregateFeatureHydrator @Inject() ( @Named(TvVideoByUserTweetCache) override val client: ReadCache[(Long, Long), DataRecord], override val statsReceiver: StatsReceiver) extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, Long), TweetCandidate] with Conditionally[ScoredTweetsQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserTweetTvVideoRealTimeAggregate") override val outputFeature: DataRecordInAFeature[TweetCandidate] = DummyFeature override def onlyIf( query: ScoredTweetsQuery ): Boolean = query.videoType.contains(hmt.VideoType.LongForm) override def features: Set[Feature[_, _]] = Set( UserTweetTvVideoRealtimeAggregatesFeatures ) override val aggregateGroups: Seq[AggregateGroup] = Seq( TvOnlineAggregationFeaturesOnlyConfig.userTweetViewRealTimeAggregates, TvOnlineAggregationFeaturesOnlyConfig.userTweetPlaybackRealTimeAggregates, TvOnlineAggregationFeaturesOnlyConfig.userTweetImpressionRealTimeAggregates ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map.empty override def keysFromQueryAndCandidates( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[(Long, Long)]] = { candidates.map(candidate => Some((query.getRequiredUserId, CandidatesUtil.getOriginalTweetId(candidate)))) } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { fetchValues(keysFromQueryAndCandidates(query, candidates)).map { featureToAggValueMaps => featureToAggValueMaps.map { featureToAggValueMap => val value = featureToAggValueMap match { case Return(r) => Some(r) case _ => None } FeatureMapBuilder() .add(UserTweetTvVideoRealtimeAggregatesFeatures, value) .build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/storehaus:core", "configapi/configapi-decider", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/module", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/thrift/src/main/thrift:thrift-scala", "media-understanding/embeddings/src/main/thrift/com/twitter/media-understanding/embeddings:thrift-scala", "servo/repo/src/main/scala", "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "strato/config/columns/content_understanding:content_understanding-strato-client", "strato/config/columns/tweetypie/managed:managed-strato-client", "strato/config/columns/user-history-transformer/unhydrated-user-history:unhydrated-user-history-strato-client", "strato/src/main/scala/com/twitter/strato/client", "user_history_transformer/common/src/main/scala/com/twitter/user_history_transformer/util", "user_history_transformer/thrift/src/main/thrift/com/twitter/user_history_transformer:user_history_transformer-scala", ], exports = [ "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/real_time_aggregates", "timelines/data_processing/ml_util/aggregation_framework/real_time_aggregates", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/BaseUserHistoryEventsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.user_history import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.BaseUserHistoryEventsAdapter import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.user_history_transformer.thriftscala.UnhydratedUserHistory import com.twitter.user_history_transformer.thriftscala.UserHistory import com.twitter.user_history_transformer.util.UserHistoryCompressionUtils import scala.collection.JavaConverters._ object UserHistoryEventsFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } trait BaseUserHistoryEventsQueryFeatureHydrator extends QueryFeatureHydrator[PipelineQuery] { private val DefaultDataRecord = new DataRecord() override def features: Set[Feature[_, _]] = Set(UserHistoryEventsFeature) def historyFetcher: Fetcher[Long, Unit, UnhydratedUserHistory] def filterEvents(query: PipelineQuery, historyEvents: Seq[UserHistory]): Stitch[Seq[UserHistory]] def adapter: BaseUserHistoryEventsAdapter override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { historyFetcher.fetch(query.getRequiredUserId).flatMap { result => result.v match { case Some(unhydratedUserHistory: UnhydratedUserHistory) => val expandedValue: UnhydratedUserHistory = UserHistoryCompressionUtils.expand(value = unhydratedUserHistory) filterEvents(query, expandedValue.events) .map { filteredEvents: Seq[UserHistory] => val record = if (filteredEvents.nonEmpty) adapter .adaptToDataRecords(filteredEvents) .asScala .headOption .getOrElse(DefaultDataRecord) else DefaultDataRecord FeatureMap(UserHistoryEventsFeature, record) } case None => Stitch.value(FeatureMap(UserHistoryEventsFeature, DefaultDataRecord)) } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/ScoredTweetsUserHistoryEventsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.user_history import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.BaseUserHistoryEventsAdapter import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.UserHistoryEventsAdapter import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.UserHistoryEventsLengthParam import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.strato.generated.client.user_history_transformer.unhydrated_user_history.UnhydratedUserHistoryMHProdClientColumn import com.twitter.user_history_transformer.thriftscala.SourceType import com.twitter.user_history_transformer.thriftscala.UnhydratedUserHistory import com.twitter.user_history_transformer.thriftscala.UserHistory import javax.inject.Inject import javax.inject.Singleton @Singleton case class ScoredTweetsUserHistoryEventsQueryFeatureHydrator @Inject() ( unhydratedUserHistoryMHProdClientColumn: UnhydratedUserHistoryMHProdClientColumn) extends BaseUserHistoryEventsQueryFeatureHydrator { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "ScoredTweetsUserHistoryEvents") override val historyFetcher: Fetcher[Long, Unit, UnhydratedUserHistory] = unhydratedUserHistoryMHProdClientColumn.fetcher private val allowedSourceTypes: Set[SourceType] = Set(SourceType.Home, SourceType.Uua) override val adapter: BaseUserHistoryEventsAdapter = UserHistoryEventsAdapter override def filterEvents( query: PipelineQuery, historyEvents: Seq[UserHistory] ): Stitch[Seq[UserHistory]] = { val filteredEvents = historyEvents.filter { event => allowedSourceTypes.contains(event.metadata.flatMap(_.source).getOrElse(SourceType.Uua)) } Stitch.value(filteredEvents.takeRight(query.params(UserHistoryEventsLengthParam))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history/ScoredVideoTweetsUserHistoryEventsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.functional_component.feature_hydrator.user_history import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.BaseUserHistoryEventsAdapter import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.VideoUserHistoryEventsAdapter import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableScoredVideoTweetsUserHistoryEventsQueryFeatureHydrationDeciderParam import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClipClusterIdInMemCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClusterId95Store import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.UserHistoryEventsLengthParam import com.twitter.mediaservices.commons.thriftscala.MediaCategory import com.twitter.mediaservices.commons.tweetmedia.thriftscala.MediaInfo import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.InProcessCache import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.strato.columns.content_understanding.thriftscala.EntityWithMetadata import com.twitter.strato.columns.content_understanding.thriftscala.TagWithMetadata import com.twitter.strato.generated.client.content_understanding.DeserializedVideoAnnotationClientColumn import com.twitter.strato.generated.client.tweetypie.managed.ImmersiveExploreClientEventsJobTesOnTweetClientColumn import com.twitter.strato.generated.client.user_history_transformer.unhydrated_user_history.UnhydratedUserHistoryMHProdClientColumn import com.twitter.unified_user_actions.thriftscala.ActionType import com.twitter.user_history_transformer.thriftscala.SourceType import com.twitter.user_history_transformer.thriftscala.UnhydratedUserHistory import com.twitter.user_history_transformer.thriftscala.UserHistory import com.twitter.user_history_transformer.thriftscala.UserHistoryMetadata import javax.inject.Inject import javax.inject.Singleton import com.twitter.tweetypie.{thriftscala => tp} import com.twitter.strato.catalog.Fetch import javax.inject.Named import com.twitter.storehaus.ReadableStore @Singleton case class ScoredVideoTweetsUserHistoryEventsQueryFeatureHydrator @Inject() ( immersiveExploreClientEventsJobTesOnTweetClientColumn: ImmersiveExploreClientEventsJobTesOnTweetClientColumn, unhydratedUserHistoryMHProdClientColumn: UnhydratedUserHistoryMHProdClientColumn, deserializedVideoAnnotationClientColumn: DeserializedVideoAnnotationClientColumn, @Named(MediaClusterId95Store) clusterIdStore: ReadableStore[Long, Long], @Named(MediaClipClusterIdInMemCache) mediaClipClusterIdInMemCache: InProcessCache[ Long, Option[Option[Long]] ], statsReceiver: StatsReceiver) extends BaseUserHistoryEventsQueryFeatureHydrator with Conditionally[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "ScoredVideoTweetsUserHistoryEvents") override def onlyIf( query: PipelineQuery ): Boolean = query.params(EnableScoredVideoTweetsUserHistoryEventsQueryFeatureHydrationDeciderParam) override val historyFetcher: Fetcher[Long, Unit, UnhydratedUserHistory] = unhydratedUserHistoryMHProdClientColumn.fetcher val tesFetcher: Fetcher[Long, tp.GetTweetFieldsOptions, tp.GetTweetFieldsResult] = immersiveExploreClientEventsJobTesOnTweetClientColumn.fetcher val TweetypieContentHydrationFields: Set[tp.TweetInclude] = Set( tp.TweetInclude.TweetFieldId(tp.Tweet.CoreDataField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.IdField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.MediaField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.MediaKeysField.id) ) private val scopedStatsReceiver: StatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val immersiveClientActionTweetIdsCounter = scopedStatsReceiver.counter("immersiveClientActionTweetIdsCounter") private val validMediaInfoCounter = scopedStatsReceiver.counter("validMediaInfoCounter") private val tesFetchCounter = statsReceiver.counter("tes_fetch_total") private val tesFetchSuccessCounter = statsReceiver.counter("tes_fetch_success") private val tesFetchFailureCounter = statsReceiver.counter("tes_fetch_failure") private val combinedTweetIdsCounter = scopedStatsReceiver.counter("combinedTweetIdsCounter") private val inValidMediaInfoCounter = scopedStatsReceiver.counter("inValidMediaInfoCounter") private val inMemCacheHitCounter = scopedStatsReceiver.counter("cache/hit") private val inMemCacheMissCounter = scopedStatsReceiver.counter("cache/miss") private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") private val clusterIdFoundCounter = scopedStatsReceiver.counter("clusterIdFoundCounter") private val clusterIdNotFoundCounter = scopedStatsReceiver.counter("clusterIdNotFoundCounter") private val clusterIdFailureCounter = scopedStatsReceiver.counter("clusterIdFailureCountere") override val adapter: BaseUserHistoryEventsAdapter = VideoUserHistoryEventsAdapter override def filterEvents( query: PipelineQuery, historyEvents: Seq[UserHistory] ): Stitch[Seq[UserHistory]] = { val immersiveActions = query match { case query: PipelineQuery with ScoredTweetsQuery => query.immersiveClientMetadata .map(metadata => metadata.immersiveClientActions) .getOrElse(Seq.empty) .filter { action => action.tweetId.isDefined && action.watchTimeMs.isDefined } case _ => Seq.empty } val immersiveActionMap: Map[Long, Int] = immersiveActions.map { action => action.tweetId.get -> action.watchTimeMs.get }.toMap val finalHistoryEvents = historyEvents .filter { event => (event.actionType == ActionType.ClientTweetVideoWatchTime || event.actionType == ActionType.ClientTweetVideoHeartbeat) && !immersiveActionMap.contains(event.tweetId) } .takeRight(query.params(UserHistoryEventsLengthParam)) val combinedTweetIds = (finalHistoryEvents.map(_.tweetId) ++ immersiveActions.map(_.tweetId.get)).distinct combinedTweetIdsCounter.incr(combinedTweetIds.size) val fetchedData: Stitch[Map[Long, tp.GetTweetFieldsResult]] = Stitch .traverse(combinedTweetIds) { tweetId => tesFetchCounter.incr() tesFetcher .fetch( tweetId, tp.GetTweetFieldsOptions(tweetIncludes = TweetypieContentHydrationFields) ) .map { case Fetch.Result(Some(resolvedResult), _) => tesFetchSuccessCounter.incr() (tweetId, resolvedResult) case _ => tesFetchFailureCounter.incr() ( tweetId, tp.GetTweetFieldsResult( tweetId, tp.TweetFieldsResultState.NotFound(tp.TweetFieldsResultNotFound()) ) ) } }.map(_.toMap) val grokMetadataFetched: Stitch[ Map[Long, (Option[Seq[TagWithMetadata]], Option[Seq[EntityWithMetadata]])] ] = Stitch .traverse(combinedTweetIds) { tweetId => deserializedVideoAnnotationClientColumn.fetcher .fetch(tweetId, Unit) .map { case Fetch.Result(response, _) => if (response.nonEmpty) keyFoundCounter.incr() else keyNotFoundCounter.incr() ( tweetId, (response.flatMap(_.tags), response.flatMap(_.entities)) ) case _ => keyFailureCounter.incr() ( tweetId, (None: Option[List[TagWithMetadata]], None: Option[List[EntityWithMetadata]]) ) } .handle { case _ => keyFailureCounter.incr() ( tweetId, (None: Option[List[TagWithMetadata]], None: Option[List[EntityWithMetadata]]) ) } }.map(_.toMap) val cluster95IdsFetched: Stitch[Map[Long, Long]] = Stitch .traverse(combinedTweetIds) { tweetId => getFromCacheOrFetch(tweetId) .map { case Some(Some(clusterId)) => clusterIdFoundCounter.incr() (tweetId, clusterId) case Some(None) | None => clusterIdNotFoundCounter.incr() (tweetId, -1L) } .handle { case _ => clusterIdFailureCounter.incr() (tweetId, -1L) } }.map(_.toMap) for { tweetResults <- fetchedData grokMetadata <- grokMetadataFetched clusterIds <- cluster95IdsFetched } yield { val tweetToMediaInfoMap = tweetResults.foldLeft( Map.empty[Long, (Long, Option[MediaCategory], Option[Int], Option[Long], Long)] ) { case (mediaInfoMap, (tweetId, tweetResult)) => tweetResult.tweetResult match { case tp.TweetFieldsResultState.Found(found) => val media = found.tweet.media.flatMap(_.headOption) val authorId = found.tweet.coreData.map(_.userId) val mediaId = media.map(_.mediaId) val mediaCategory = media.flatMap(_.mediaKey).map(_.mediaCategory) val videoDuration = media.flatMap(_.mediaInfo match { case Some(MediaInfo.VideoInfo(videoInfo)) => Some(videoInfo.durationMillis) case _ => None }) val clusterId = clusterIds.getOrElse(tweetId, -1L) mediaId match { case Some(id) => mediaInfoMap + (tweetId -> ( ( id, mediaCategory, videoDuration, authorId, clusterId ) )) case None => mediaInfoMap } case _ => mediaInfoMap } } val immersiveActionsUserHistory = immersiveActions.map { action => val tweetId = action.tweetId.get val (tagsOpt, entitiesOpt) = grokMetadata.getOrElse(tweetId, (None, None)) val mediaInfo = tweetToMediaInfoMap.get(tweetId) val finalTags = tagsOpt.getOrElse(List.empty[TagWithMetadata]) val finalEntities = entitiesOpt.getOrElse(List.empty[EntityWithMetadata]) val finalAuthorId: Option[Long] = mediaInfo.flatMap(_._4) val finalClusterId: Long = mediaInfo.map(_._5).getOrElse(-1L) val (finalMediaId, finalMediaCategory, finalVideoDuration) = mediaInfo match { case Some((mediaId, mediaCategory, videoDuration, _, _)) => validMediaInfoCounter.incr() (Some(mediaId), mediaCategory, videoDuration) case None => inValidMediaInfoCounter.incr() (Some(-1L), Some(MediaCategory.TweetVideo), Some(-1)) } UserHistory( userId = query.getRequiredUserId, tweetId = tweetId, authorId = finalAuthorId, actionType = ActionType.ClientTweetVideoWatchTime, engagedTimestampMs = query.queryTime.inMilliseconds, textTokens = None, sourceTweetId = Some(tweetId), metadata = Some( UserHistoryMetadata( source = Some(SourceType.ImmersiveVideo), watchTime = Some(action.watchTimeMs.get.toDouble), mediaId = finalMediaId, mediaCategory = finalMediaCategory, videoDuration = finalVideoDuration, tags = Some(finalTags), entities = Some(finalEntities), clusterId = Some(finalClusterId) ) ) ) } immersiveClientActionTweetIdsCounter.incr(immersiveActionsUserHistory.size) (finalHistoryEvents ++ immersiveActionsUserHistory).map { event => val tweetId = event.tweetId val (tagsOpt, entitiesOpt) = grokMetadata.getOrElse(tweetId, (None, None)) val mediaInfo = tweetToMediaInfoMap.get(tweetId) val finalTags = tagsOpt.getOrElse(List.empty[TagWithMetadata]) val finalEntities = entitiesOpt.getOrElse(List.empty[EntityWithMetadata]) val finalAuthorId: Option[Long] = mediaInfo.flatMap(_._4) val finalClusterId: Long = mediaInfo.map(_._5).getOrElse(-1L) val (finalMediaId, finalMediaCategory, finalVideoDuration) = mediaInfo match { case Some((mediaId, mediaCategory, videoDuration, _, _)) => validMediaInfoCounter.incr() (Some(mediaId), mediaCategory, videoDuration) case None => inValidMediaInfoCounter.incr() (Some(-1L), Some(MediaCategory.TweetVideo), Some(-1)) } event.copy( authorId = finalAuthorId.orElse(event.authorId), metadata = event.metadata.map { metadata => metadata.copy( mediaId = finalMediaId, mediaCategory = finalMediaCategory, videoDuration = finalVideoDuration, tags = Some(finalTags), entities = Some(finalEntities), clusterId = Some(finalClusterId) ) } ) } } } private def getFromCacheOrFetch(tweetId: Long): Stitch[Option[Option[Long]]] = { mediaClipClusterIdInMemCache .get(tweetId) .map { cachedValue => inMemCacheHitCounter.incr() Stitch.value(cachedValue) }.getOrElse { inMemCacheMissCounter.incr() Stitch .callFuture(clusterIdStore.get(tweetId)) .map { result => val finalResult = Some(result) mediaClipClusterIdInMemCache.set(tweetId, finalResult) finalResult } .handle { case _ => None } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/AuthorDedupFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Keep only 1 tweet per author */ object AuthorDedupFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("AuthorDedup") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val keptIds = candidates .groupBy { candidate => candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L) }.map { case (_, candidates) => candidates.head.candidate.id }.toSet val (kept, removed) = candidates.partition(c => keptIds.contains(c.candidate.id)) Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/transformer_embeddings", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/location", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", "stitch/stitch-socialgraph", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", "util/util-slf4j-api/src/main/scala", ], exports = [ "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ClipClusterDeduplicationFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.ClipImageClusterIdsFeature import com.twitter.home_mixer.model.HomeFeatures.TweetMediaClusterIdsFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch trait ClipClusterDeduplicationFilter extends Filter[PipelineQuery, TweetCandidate] { val clusterIdFeature: Feature[TweetCandidate, Map[Long, Long]] override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (removedCandidateIds, _) = candidates.foldLeft((Set.empty[Long], Set.empty[Long])) { case ((removedIds, seenClusterIds), candidate) => val clusterIds = candidate.features .getOrElse(clusterIdFeature, Map.empty[Long, Long]) .values .toSet if (clusterIds.size == 1) { val clusterId = clusterIds.head if (seenClusterIds.contains(clusterId)) { (removedIds + candidate.candidate.id, seenClusterIds) } else { (removedIds, seenClusterIds + clusterId) } } else { (removedIds, seenClusterIds) } } val (removed, kept) = candidates .map(_.candidate) .partition(c => removedCandidateIds.contains(c.id)) Stitch.value(FilterResult(kept = kept, removed = removed)) } } object ClipImageClusterDeduplicationFilter extends ClipClusterDeduplicationFilter { override val identifier: FilterIdentifier = FilterIdentifier("ClipImageClusterDeduplication") override val clusterIdFeature: Feature[TweetCandidate, Map[Long, Long]] = ClipImageClusterIdsFeature } object ClipVideoClusterDeduplicationFilter extends ClipClusterDeduplicationFilter { override val identifier: FilterIdentifier = FilterIdentifier("ClipVideoClusterDeduplication") override val clusterIdFeature: Feature[TweetCandidate, Map[Long, Long]] = TweetMediaClusterIdsFeature } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ClusterBasedDedupFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.UserHistoryEventsFeatures import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.transformer_embeddings.VideoUserHistoryEventsFeatures import com.twitter.home_mixer.functional_component.feature_hydrator.user_history.UserHistoryEventsFeature import com.twitter.home_mixer.model.HomeFeatures.DedupClusterId88Feature import com.twitter.home_mixer.model.HomeFeatures.DedupClusterIdFeature import com.twitter.home_mixer.param.HomeGlobalParams.DedupHistoricalEventsTimeWindowParam import com.twitter.home_mixer.param.HomeGlobalParams.EnableClusterBased88DedupFilter import com.twitter.home_mixer.param.HomeGlobalParams.EnableClusterBasedDedupFilter import com.twitter.home_mixer.param.HomeGlobalParams.EnableNoClusterFilter import scala.collection.convert.ImplicitConversions._ import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object ClusterBasedDedupFilter extends Filter[PipelineQuery, TweetCandidate] with Filter.Conditionally[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("ClusterBasedDedup") override def onlyIf( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Boolean = { query.params(EnableClusterBasedDedupFilter) } private def getViewedClusterIds(query: PipelineQuery): Set[Long] = { val currentTimeMs = System.currentTimeMillis() val dedupHistoricalEventsTimeWindow = query.params(DedupHistoricalEventsTimeWindowParam) val dataRecord = query.features.get .getOrElse(UserHistoryEventsFeature, new DataRecord()) val clusterIds = dataRecord.getTensors match { case tensors if tensors != null => val clusterIdsTensor = tensors.get(VideoUserHistoryEventsFeatures.VideoCluster95IdsFeature.getFeatureId) val actionTimestampTensor = tensors.get(UserHistoryEventsFeatures.ActionTimestampsFeature.getFeatureId) if (clusterIdsTensor != null && clusterIdsTensor.isSet() && actionTimestampTensor != null && actionTimestampTensor.isSet()) { val actionTimestampsBuffer = actionTimestampTensor.getInt64Tensor.longs .map(_.longValue()) val clusterIdsBuffer = clusterIdsTensor.getInt64Tensor.longs.map(_.longValue()) clusterIdsBuffer .zip(actionTimestampsBuffer) .collect { case (clusterId, timestamp) if timestamp > 0 && (currentTimeMs - timestamp) <= dedupHistoricalEventsTimeWindow => clusterId } .toSet } else { Set.empty[Long] } case _ => Set.empty[Long] } clusterIds } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val historicalClusters = getViewedClusterIds(query) // Still based on 0.95 threshold val enable88Dedup = query.params(EnableClusterBased88DedupFilter) val enableNoClusterFilter = query.params(EnableNoClusterFilter) val (kept, removed) = candidates.foldLeft( ( Seq[CandidateWithFeatures[TweetCandidate]](), // kept accumulator Seq[CandidateWithFeatures[TweetCandidate]](), // removed accumulator Set[Long](), // seen cluster IDs for 0.95 threshold Set[Long]() // seen cluster IDs for 0.88 threshold ) ) { case ((keptAcc, removedAcc, seenCluster95Ids, seenCluster88Ids), candidate) => val clusterId95 = candidate.features .getOrElse(DedupClusterIdFeature, None) // 0.95 threshold val clusterId88 = candidate.features .getOrElse(DedupClusterId88Feature, None) // 0.88 threshold (clusterId95, clusterId88) match { case (Some(c95), _) if historicalClusters.contains(c95) => // Historical match at 0.95 threshold, remove it (keptAcc, removedAcc :+ candidate, seenCluster95Ids, seenCluster88Ids) case (Some(c95), _) if seenCluster95Ids.contains(c95) => // Duplicate at 0.95 threshold (non-historical), remove it (keptAcc, removedAcc :+ candidate, seenCluster95Ids, seenCluster88Ids) case (Some(c95), Some(c88)) if enable88Dedup && seenCluster88Ids.contains(c88) => // Unique at 0.95 but duplicate at 0.88 threshold, remove it (keptAcc, removedAcc :+ candidate, seenCluster95Ids + c95, seenCluster88Ids) case (Some(c95), Some(c88)) => // First occurrence at both levels, keep it val updatedSeen88Ids = if (enable88Dedup) seenCluster88Ids + c88 else seenCluster88Ids (keptAcc :+ candidate, removedAcc, seenCluster95Ids + c95, updatedSeen88Ids) case (Some(c95), None) => // Only 0.95 cluster ID, keep it if not seen (keptAcc :+ candidate, removedAcc, seenCluster95Ids + c95, seenCluster88Ids) case (None, Some(c88)) if enable88Dedup && seenCluster88Ids.contains(c88) => // Only 0.88 cluster ID, remove if duplicate (keptAcc, removedAcc :+ candidate, seenCluster95Ids, seenCluster88Ids) case (None, Some(c88)) => // Only 0.88 cluster ID, keep if not seen val updatedSeen88Ids = if (enable88Dedup) seenCluster88Ids + c88 else seenCluster88Ids (keptAcc :+ candidate, removedAcc, seenCluster95Ids, updatedSeen88Ids) case (None, None) => // No cluster IDs, keep or remove based on enableNoClusterFilter if (enableNoClusterFilter) { (keptAcc, removedAcc :+ candidate, seenCluster95Ids, seenCluster88Ids) // Remove it } else { (keptAcc :+ candidate, removedAcc, seenCluster95Ids, seenCluster88Ids) // Keep it } } } match { case (keptCandidates, removedCandidates, _, _) => (keptCandidates, removedCandidates) } Stitch.value( FilterResult( kept = kept.map(_.candidate), removed = removed.map(_.candidate) ) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ConsistentAspectRatioFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.VideoAspectRatioFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.Param /** * Filter for making sure all video posts in a carousel has consistent aspect ratio */ case class ConsistentAspectRatioFilter( allowVerticalVideosParam: Param[Boolean], allowHorizontalVideosParam: Param[Boolean]) extends Filter[PipelineQuery, TweetCandidate] { /** @see [[FilterIdentifier]] */ override val identifier: FilterIdentifier = FilterIdentifier("ConsistentAspectRatio") /** * Filter the list of candidates * * @return a FilterResult including both the list of kept candidate and the list of removed candidates */ override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (hasAspectRatio, doesNotHaveAspectRatio) = candidates .partition(_.features.getOrElse(VideoAspectRatioFeature, None).nonEmpty) if (hasAspectRatio.nonEmpty) { val isHorizontal = (query.params(allowVerticalVideosParam), query.params(allowHorizontalVideosParam)) match { case (false, _) => false case (_, false) => true case _ => // If both video types are allowed, take the first video's aspect ratio val aspectRatioFirstVideo = hasAspectRatio.head.features.getOrElse(VideoAspectRatioFeature, None).get aspectRatioFirstVideo > 1.0 } val (consistentAspectRatio, differentAspectRatio) = hasAspectRatio.partition { candidate => candidate.features.getOrElse(VideoAspectRatioFeature, None).get > 1.0 == isHorizontal } Stitch.value( FilterResult( kept = consistentAspectRatio.map(_.candidate), removed = ((differentAspectRatio ++ doesNotHaveAspectRatio)).map(_.candidate))) } else { Stitch.value( FilterResult(kept = Seq.empty, removed = doesNotHaveAspectRatio.map(_.candidate))) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/CountryFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.home_mixer.param.HomeGlobalParams.EnableCountryFilter /** * Filters tweets based on matching country location between user query and tweet candidates */ object CountryFilter extends Filter[PipelineQuery, TweetCandidate] with Filter.Conditionally[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("Country") /** * Determines if the filter should be applied based on the EnableCountryFilter parameter */ override def onlyIf( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Boolean = { query.params(EnableCountryFilter) } /** * Filters candidates based on exact country location matching * @param query The pipeline query containing user location features * @param candidates The sequence of tweet candidates with their features * @return A Stitch containing FilterResult with kept and removed candidates */ override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val userCountryOpt = query.features .map(_.getOrElse(LocationFeature, None)) .flatMap(_.flatMap(_.country)) val (kept, removed) = candidates.partition { candidate => val postLocationOpt = candidate.features.getOrElse(LocationFeature, None).flatMap(_.country) val keep = (postLocationOpt, userCountryOpt) match { case (Some(postLocation), Some(userCountry)) => postLocation == userCountry case (Some(_), None) => false case (None, _) => true case _ => true } keep } Stitch.value( FilterResult( kept = kept.map(_.candidate), removed = removed.map(_.candidate) )) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/CurrentPinnedTweetFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.CurrentPinnedTweetFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Keep only the currently pinned tweet per author */ object CurrentPinnedTweetFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("CurrentPinnedTweet") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (kept, removed) = candidates.partition { candidate => val currentPinnedTweet = candidate.features.getOrElse(CurrentPinnedTweetFeature, None) currentPinnedTweet.contains(candidate.candidate.id) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/DropMaxCandidatesFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.FSBoundedParam case class DropMaxCandidatesFilter[Candidate <: UniversalNoun[Any]]( maxCandidatesParam: FSBoundedParam[Int]) extends Filter[PipelineQuery, Candidate] { override val identifier: FilterIdentifier = FilterIdentifier("DropMaxCandidates") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[Candidate]] ): Stitch[FilterResult[Candidate]] = { val maxCandidates = query.params(maxCandidatesParam) val (kept, removed) = candidates.map(_.candidate).splitAt(maxCandidates) Stitch.value(FilterResult(kept, removed)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature import com.twitter.home_mixer.param.HomeGlobalParams.FeedbackFatigueFilteringDurationParam import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.common.thriftscala.FeedbackEntity import com.twitter.timelineservice.model.FeedbackEntry import com.twitter.timelineservice.{thriftscala => tls} object FeedbackFatigueFilter extends Filter[PipelineQuery, TweetCandidate] with Filter.Conditionally[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("FeedbackFatigue") override def onlyIf( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Boolean = query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty) override def apply( query: pipeline.PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val feedbackEntriesByEngagementType = query.features .getOrElse(FeatureMap.empty).getOrElse(FeedbackHistoryFeature, Seq.empty) .filter { entry => val timeSinceFeedback = query.queryTime.minus(entry.timestamp) timeSinceFeedback < query.params(FeedbackFatigueFilteringDurationParam) && entry.feedbackType == tls.FeedbackType.SeeFewer }.groupBy(_.engagementType) val authorsToFilter = getUserIds( feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Tweet, Seq.empty)) val likersToFilter = getUserIds( feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Like, Seq.empty)) val followersToFilter = getUserIds( feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Follow, Seq.empty)) val retweetersToFilter = getUserIds( feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty)) val (removed, kept) = candidates.partition { candidate => val originalAuthorId = CandidatesUtil.getOriginalAuthorId(candidate.features) val authorId = candidate.features.getOrElse(AuthorIdFeature, None) val likers = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) val eligibleLikers = likers.filterNot(likersToFilter.contains) val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty) val eligibleFollowers = followers.filterNot(followersToFilter.contains) originalAuthorId.exists(authorsToFilter.contains) || (likers.nonEmpty && eligibleLikers.isEmpty) || (followers.nonEmpty && eligibleFollowers.isEmpty && likers.isEmpty) || (candidate.features.getOrElse(IsRetweetFeature, false) && authorId.exists(retweetersToFilter.contains)) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } private def getUserIds( feedbackEntries: Seq[FeedbackEntry], ): Set[Long] = feedbackEntries.collect { case FeedbackEntry(_, _, FeedbackEntity.UserId(userId), _, _, _) => userId }.toSet } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokGoreFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.param.HomeGlobalParams.EnableGrokGoreFilter import com.twitter.home_mixer.model.HomeFeatures.GrokAnnotationsFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Filter out gore tweets based on grok annotations */ object GrokGoreFilter extends Filter[PipelineQuery, TweetCandidate] with Filter.Conditionally[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("GrokGore") override def onlyIf( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Boolean = query.params(EnableGrokGoreFilter) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (removed, kept) = candidates.partition { candidate => val annotations = candidate.features.getOrElse(GrokAnnotationsFeature, None) annotations.flatMap(_.metadata.map(_.isGore)).getOrElse(false) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokNsfwFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.param.HomeGlobalParams.EnableNsfwFilter import com.twitter.home_mixer.param.HomeGlobalParams.EnableSoftNsfwFilter import com.twitter.home_mixer.model.HomeFeatures.GrokAnnotationsFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Filter out NSFW tweets based on grok annotations */ object GrokNsfwFilter extends Filter[PipelineQuery, TweetCandidate] with Filter.Conditionally[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("GrokNsfw") override def onlyIf( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Boolean = query.params(EnableNsfwFilter) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (removed, kept) = candidates.partition { candidate => val annotations = candidate.features.getOrElse(GrokAnnotationsFeature, None) val isNsfw = annotations.flatMap(_.metadata.map(_.isNsfw)).getOrElse(false) val isSoftNsfw = annotations.flatMap(_.metadata.map(_.isSoftNsfw)).getOrElse(false) if (query.params(EnableSoftNsfwFilter)) { isNsfw || isSoftNsfw } else { isNsfw } } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokSpamFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.param.HomeGlobalParams.EnableGrokSpamFilter import com.twitter.home_mixer.model.HomeFeatures.GrokAnnotationsFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Filter out spam tweets based on grok annotations */ object GrokSpamFilter extends Filter[PipelineQuery, TweetCandidate] with Filter.Conditionally[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("GrokSpam") override def onlyIf( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Boolean = query.params(EnableGrokSpamFilter) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (removed, kept) = candidates.partition { candidate => val annotations = candidate.features.getOrElse(GrokAnnotationsFeature, None) annotations.flatMap(_.metadata.map(_.isSpam)).getOrElse(false) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/GrokViolentFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.param.HomeGlobalParams.EnableGrokViolentFilter import com.twitter.home_mixer.model.HomeFeatures.GrokAnnotationsFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Filter out violent tweets based on grok annotations */ object GrokViolentFilter extends Filter[PipelineQuery, TweetCandidate] with Filter.Conditionally[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("GrokViolent") override def onlyIf( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Boolean = query.params(EnableGrokViolentFilter) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (removed, kept) = candidates.partition { candidate => val annotations = candidate.features.getOrElse(GrokAnnotationsFeature, None) annotations.flatMap(_.metadata.map(_.isViolent)).getOrElse(false) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/HasAuthorFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object HasAuthorFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("HasAuthor") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (kept, removed) = candidates.partition { candidate => candidate.features.getOrElse(AuthorIdFeature, None).isDefined } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/HasMultipleMediaFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.HasMultipleMedia import com.twitter.home_mixer.param.HomeGlobalParams.EnableHasMultipleMediaFilter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object HasMultipleMediaFilter extends Filter[PipelineQuery, TweetCandidate] with Filter.Conditionally[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("HasMultipleMedia") override def onlyIf( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Boolean = { query.params(EnableHasMultipleMediaFilter) } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (removed, kept) = candidates.partition { candidate => candidate.features.getOrElse(HasMultipleMedia, false) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidConversationModuleFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Exclude conversation modules where Tweets have been dropped by other filters * * Largest conversation modules have 3 Tweets, so if all 3 are present, module is valid. * For 2 Tweet modules, check if the head is the root (not a reply) and the last item * is actually replying to the root directly with no missing intermediate tweets */ object InvalidConversationModuleFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("InvalidConversationModule") val ValidThreeTweetModuleSize = 3 val ValidTwoTweetModuleSize = 2 override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val allowedTweetIds = candidates .groupBy(_.features.getOrElse(ConversationModuleFocalTweetIdFeature, None)) .map { case (id, candidates) => (id, candidates.sortBy(_.candidate.id)) } .filter { case (Some(_), conversation) if conversation.size == ValidThreeTweetModuleSize => true case (Some(focalId), conversation) if conversation.size == ValidTwoTweetModuleSize => conversation.head.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty && conversation.last.candidate.id == focalId && conversation.last.features .getOrElse(InReplyToTweetIdFeature, None) .contains(conversation.head.candidate.id) case (None, _) => true case _ => false }.values.flatten.toSeq.map(_.candidate.id).toSet val (kept, removed) = candidates.map(_.candidate).partition(candidate => allowedTweetIds.contains(candidate.id)) Stitch.value(FilterResult(kept = kept, removed = removed)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.tracing.Trace import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.socialgraph.{thriftscala => sg} import com.twitter.stitch.Stitch import com.twitter.stitch.socialgraph.SocialGraph import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton import scala.collection.immutable.ListSet /** * Exclude invalid subscription tweets - cases where the viewer is not subscribed to the author * * If SGS hydration fails, `SGSInvalidSubscriptionTweetFeature` will be set to None for * subscription tweets, so we explicitly filter those tweets out. */ @Singleton case class InvalidSubscriptionTweetFilter @Inject() ( socialGraphClient: SocialGraph, statsReceiver: StatsReceiver) extends Filter[PipelineQuery, TweetCandidate] with Logging { override val identifier: FilterIdentifier = FilterIdentifier("InvalidSubscriptionTweet") private val scopedStatsReceiver = statsReceiver.scope(identifier.toString) private val servedTypeStatsReceiver = scopedStatsReceiver.scope("ServedType") private val validCounter = scopedStatsReceiver.counter("validExclusiveTweet") private val invalidCounter = scopedStatsReceiver.counter("invalidExclusiveTweet") private val forYouScoredTweetsCandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouScoredTweets") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = Stitch .traverse(candidates) { candidate => val exclusiveAuthorId = candidate.features.getOrElse(ExclusiveConversationAuthorIdFeature, None) if (exclusiveAuthorId.isDefined) { val request = sg.ExistsRequest( source = query.getRequiredUserId, target = exclusiveAuthorId.get, relationships = Seq(sg.Relationship(sg.RelationshipType.TierOneSuperFollowing, hasRelationship = true)), ) socialGraphClient.exists(request).map(_.exists).map { valid => if (!valid) { invalidCounter.incr() val candidatePipelines = candidate.features .getOrElse(CandidatePipelines, ListSet.empty[CandidatePipelineIdentifier]) // Temporary debugging code if (candidatePipelines.contains(forYouScoredTweetsCandidatePipelineIdentifier)) { val servedType = candidate.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined) servedTypeStatsReceiver.counter(servedType.name).incr() logger.info( s"Removing subscription Tweet ${candidate.candidate.id} " + s"for User ${query.getRequiredUserId} from Author $exclusiveAuthorId " + s"with traceId: ${Trace.id.traceId.toLong}" ) } } else validCounter.incr() valid } } else Stitch.value(true) }.map { validResults => val (kept, removed) = candidates .map(_.candidate) .zip(validResults) .partition { case (candidate, valid) => valid } val keptCandidates = kept.map { case (candidate, _) => candidate } val removedCandidates = removed.map { case (candidate, _) => candidate } FilterResult(kept = keptCandidates, removed = removedCandidates) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/LocationFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationFeature import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationIdFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object LocationFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("Location") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val userLocationOpt = query.features.get.getOrElse(LocationFeature, None) val (kept, removed) = candidates.partition { candidate => val postLocationOpt = candidate.features.getOrElse(LocationIdFeature, None) (postLocationOpt, userLocationOpt) match { case (Some(postLocation), Some(userLocation)) => userLocation.matches(postLocation) case (Some(postLocation), None) => false case _ => true } } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/MaxVideoDurationFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableMaxVideoDurationFilter import com.twitter.home_mixer.param.HomeGlobalParams.MaxVideoDurationThresholdParam import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Filter out tweets based on video duration */ object MaxVideoDurationFilter extends Filter[PipelineQuery, TweetCandidate] with Filter.Conditionally[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("MaxVideoDuration") override def onlyIf( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Boolean = query.params(EnableMaxVideoDurationFilter) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val maxVideoDurationThresh = query.params(MaxVideoDurationThresholdParam) val (kept, removed) = candidates.partition { candidate => val hasVideoFeature = candidate.features.getOrElse(HasVideoFeature, false) val videoDuration = candidate.features.getOrElse(VideoDurationMsFeature, None).getOrElse(0) hasVideoFeature && (videoDuration <= maxVideoDurationThresh) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/MediaDeduplicationFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Remove duplicate media in the same payload */ object MediaIdDeduplicationFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("MediaDeduplication") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (removedCandidateIds, _) = candidates.foldLeft((Set.empty[Long], Set.empty[Long])) { case ((removedIds, seenMediaIds), candidate) => val mediaIds = candidate.features.getOrElse(TweetMediaIdsFeature, Seq.empty[Long]) if (mediaIds.size == 1) { val mediaId = mediaIds.head if (seenMediaIds.contains(mediaId)) { (removedIds + candidate.candidate.id, seenMediaIds) } else { (removedIds, seenMediaIds + mediaId) } } else { (removedIds, seenMediaIds) } } val (removed, kept) = candidates.map(_.candidate) .partition(c => removedCandidateIds.contains(c.id)) Stitch.value(FilterResult(kept = kept, removed = removed)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/MinVideoDurationFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.TweetMediaCompletionRateFeature import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableMinVideoDurationFilter import com.twitter.home_mixer.param.HomeGlobalParams.MinVideoDurationThresholdParam import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Filter out tweets based on video duration */ object MinVideoDurationFilter extends Filter[PipelineQuery, TweetCandidate] with Filter.Conditionally[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("MinVideoDuration") override def onlyIf( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Boolean = query.params(EnableMinVideoDurationFilter) private val CompletionRateThreshold = 90 private val FavThreshold = 30 private val LongDuration = 5000 override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val minVideoDurationThresh = query.params(MinVideoDurationThresholdParam) val (removed, kept) = candidates.partition { candidate => val hasVideoFeature = candidate.features.getOrElse(HasVideoFeature, false) val completionRate = candidate.features.getOrElse(TweetMediaCompletionRateFeature, None).getOrElse(0.0) val videoDuration = candidate.features.getOrElse(VideoDurationMsFeature, None).getOrElse(0) val ebFeatures = candidate.features.getOrElse(EarlybirdFeature, None) val favCount = ebFeatures.flatMap(_.favCountV2) val sketchyCompletionRate = favCount.forall(_ > FavThreshold) && completionRate > CompletionRateThreshold && videoDuration > LongDuration val lowDuration = videoDuration <= minVideoDurationThresh hasVideoFeature && (lowDuration || sketchyCompletionRate) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateGatedFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.product_mixer.core.functional_component.common.alert.Alert import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch trait FilterPredicate[-Query <: PipelineQuery] { def apply(query: Query): Boolean } /** * A [[Filter]] with [[Conditionally]] based on a [[FilterPredicate]] * * @param predicate the predicate to turn this filter on and off * @param filter the underlying filter to run when `predicate` is true * @tparam Query The domain model for the query or request * @tparam Candidate The type of the candidates */ case class PredicateGatedFilter[-Query <: PipelineQuery, Candidate <: UniversalNoun[Any]]( predicate: FilterPredicate[Query], filter: Filter[Query, Candidate]) extends Filter[Query, Candidate] with Filter.Conditionally[Query, Candidate] { override val identifier: FilterIdentifier = FilterIdentifier( PredicateGatedFilter.IdentifierPrefix + filter.identifier.name) override val alerts: Seq[Alert] = filter.alerts override def onlyIf(query: Query, candidates: Seq[CandidateWithFeatures[Candidate]]): Boolean = Conditionally.and(Filter.Input(query, candidates), filter, predicate(query)) override def apply( query: Query, candidates: Seq[CandidateWithFeatures[Candidate]] ): Stitch[FilterResult[Candidate]] = filter.apply(query, candidates) } object PredicateGatedFilter { val IdentifierPrefix = "PredicateGated" } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenMediaIdsFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.ImpressedMediaIds import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Filter out users' previously seen mediaIds */ object PreviouslySeenMediaIdsFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("PreviouslySeenMediaIds") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val seenMediaIds = query.features.map(_.getOrElse(ImpressedMediaIds, Seq.empty)).toSet.flatten val (removed, kept) = candidates.partition { candidate => candidate.features.getOrElse(TweetMediaIdsFeature, Seq.empty).exists(seenMediaIds.contains) } val filterResult = FilterResult( kept = kept.map(_.candidate), removed = removed.map(_.candidate) ) Stitch.value(filterResult) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslySeenTweetsFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature import com.twitter.home_mixer.model.HomeFeatures.UserRecentEngagementTweetIdsFeature import com.twitter.home_mixer.model.request.HasSeenTweetIds import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilterItem /** * Filter out users' previously seen tweets from Impression Bloom Filter */ object PreviouslySeenTweetsFilter extends Filter[PipelineQuery with HasSeenTweetIds, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("PreviouslySeenTweets") override def apply( query: PipelineQuery with HasSeenTweetIds, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val bloomFilterSeq = query.features.map(_.get(ImpressionBloomFilterFeature)).get val bloomFilters = bloomFilterSeq.entries.map(ImpressionBloomFilterItem.fromThrift(_).bloomFilter) val seenTweetIds = query.seenTweetIds.getOrElse(Seq.empty).toSet val engagedTweetIds = query.features .map(_.getOrElse(UserRecentEngagementTweetIdsFeature, Seq.empty).toSet) .getOrElse(Set.empty) val (removed, kept) = candidates.partition { candidate => CandidatesUtil.getTweetIdAndSourceId(candidate).exists { tweetId => seenTweetIds.contains(tweetId) || engagedTweetIds.contains(tweetId) || bloomFilters.exists(filter => filter.mayContain(tweetId)) } } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedAncestorsFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent import com.twitter.home_mixer.model.HomeFeatures.IsAncestorCandidateFeature import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelinemixer.injection.store.persistence.TimelinePersistenceUtils import com.twitter.timelines.util.client_info.ClientPlatform object PreviouslyServedAncestorsFilter extends Filter[PipelineQuery, TweetCandidate] with TimelinePersistenceUtils { override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedAncestors") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val clientPlatform = ClientPlatform.fromQueryOptions( clientAppId = query.clientContext.appId, userAgent = query.clientContext.userAgent.flatMap(UserAgent.fromString)) val entries = query.features.map(_.getOrElse(PersistenceEntriesFeature, Seq.empty)).toSeq.flatten val tweetIds = applicableResponses(clientPlatform, entries) .flatMap(_.entries.flatMap(_.tweetIds(includeSourceTweets = true))).toSet val ancestorIds = candidates .filter(_.features.getOrElse(IsAncestorCandidateFeature, false)).map(_.candidate.id).toSet val (removed, kept) = candidates .map(_.candidate).partition(candidate => tweetIds.contains(candidate.id) && ancestorIds.contains(candidate.id)) Stitch.value(FilterResult(kept = kept, removed = removed)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object PreviouslyServedTweetPreviewsFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedTweetPreviews") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val servedTweetPreviewIds = query.features.map(_.getOrElse(ServedTweetPreviewIdsFeature, Seq.empty)).toSeq.flatten.toSet val (removed, kept) = candidates.partition { candidate => servedTweetPreviewIds.contains(candidate.candidate.id) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetsFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableServedFilterAllRequests import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object PreviouslyServedTweetsFilter extends Filter[PipelineQuery, TweetCandidate] with Filter.Conditionally[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedTweets") override def onlyIf( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Boolean = { query.features.exists(_.getOrElse(GetOlderFeature, false)) || query.params(EnableServedFilterAllRequests) } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val servedTweetIds = query.features.map(_.getOrElse(ServedTweetIdsFeature, Seq.empty)).toSeq.flatten.toSet val (removed, kept) = candidates.partition { candidate => val tweetIdAndSourceId = CandidatesUtil.getTweetIdAndSourceId(candidate) tweetIdAndSourceId.exists(servedTweetIds.contains) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/QuoteDeduplicationFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Remove base tweet if it is quoted in another candidate */ object QuoteDeduplicationFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("QuoteDeduplication") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val quotedTweets = candidates.flatMap(_.features.getOrElse(QuotedTweetIdFeature, None)).toSet val (removed, kept) = candidates.map(_.candidate).partition(candidate => quotedTweets.contains(candidate.id)) Stitch.value(FilterResult(kept = kept, removed = removed)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RegionFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.home_mixer.param.HomeGlobalParams.EnableRegionFilter /** * Filters tweets based on matching region location between user query and tweet candidates */ object RegionFilter extends Filter[PipelineQuery, TweetCandidate] with Filter.Conditionally[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("Region") override def onlyIf( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Boolean = { query.params(EnableRegionFilter) } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val userRegionOpt = query.features .map(_.getOrElse(LocationFeature, None)) .flatMap(_.flatMap(_.region)) val (kept, removed) = candidates.partition { candidate => val postLocationOpt = candidate.features.getOrElse(LocationFeature, None).flatMap(_.region) val keep = (postLocationOpt, userRegionOpt) match { case (Some(postLocation), Some(userRegion)) => postLocation == userRegion case (Some(_), None) => false case (None, _) => true case _ => true } keep } Stitch.value( FilterResult( kept = kept.map(_.candidate), removed = removed.map(_.candidate) )) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RejectTweetFromViewerFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object RejectTweetFromViewerFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("RejectTweetFromViewer") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (removed, kept) = candidates.partition(candidate => CandidatesUtil.isAuthoredByViewer(query, candidate.features)) Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object ReplyFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("Reply") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (kept, removed) = candidates.partition { candidate => candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetDeduplicationFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import scala.collection.mutable object RetweetDeduplicationFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("RetweetDeduplication") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { // If there are 2 retweets of the same native tweet, we will choose the first one // The tweets are returned in descending score order, so we will choose the higher scored tweet val dedupedTweetIdsSet = candidates.partition(_.features.getOrElse(IsRetweetFeature, false)) match { case (retweets, nativeTweets) => val nativeTweetIds = nativeTweets.map(_.candidate.id) val seenTweetIds = mutable.Set[Long]() ++ nativeTweetIds val dedupedRetweets = retweets.filter { retweet => val tweetIdAndSourceId = CandidatesUtil.getTweetIdAndSourceId(retweet) val retweetIsUnique = tweetIdAndSourceId.forall(!seenTweetIds.contains(_)) if (retweetIsUnique) { seenTweetIds ++= tweetIdAndSourceId } retweetIsUnique } (nativeTweets ++ dedupedRetweets).map(_.candidate.id).toSet } val (kept, removed) = candidates.map(_.candidate).partition(candidate => dedupedTweetIdsSet.contains(candidate.id)) Stitch.value(FilterResult(kept = kept, removed = removed)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object RetweetFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("Retweet") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (kept, removed) = candidates .partition { candidate => !candidate.features.getOrElse(IsRetweetFeature, false) } val filterResult = FilterResult( kept = kept.map(_.candidate), removed = removed.map(_.candidate) ) Stitch.value(filterResult) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/SlopFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.core_workflows.user_model.{thriftscala => um} import com.twitter.home_mixer.model.HomeFeatures.AuthorFollowersFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.SlopAuthorFeature import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableSlopFilter import com.twitter.home_mixer.param.HomeGlobalParams.EnableSlopFilterEligibleUserStateParam import com.twitter.home_mixer.param.HomeGlobalParams.EnableSlopFilterLowSignalUsers import com.twitter.home_mixer.param.HomeGlobalParams.SlopMinFollowers import com.twitter.home_mixer.util.SignalUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object SlopFilter extends Filter[PipelineQuery, TweetCandidate] with Filter.Conditionally[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("Slop") private val MinFollowingThreshold = 5 private val EligibleUserStates: Set[um.UserState] = Set(um.UserState.NearZero, um.UserState.New, um.UserState.VeryLight) override def onlyIf( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Boolean = { val numSlopAuthorsFollowed = candidates .filter { candidate => candidate.features.getOrElse(SlopAuthorFeature, false) && candidate.features.getOrElse(InNetworkFeature, false) }.flatMap(_.features.getOrElse(AuthorIdFeature, None)).distinct.size val userState = query.features.flatMap(_.getOrElse(UserStateFeature, None)) val lowSignalUser = SignalUtil.isLowSignalUser(query) numSlopAuthorsFollowed < MinFollowingThreshold && ( (userState.forall(EligibleUserStates.contains) && query.params(EnableSlopFilterEligibleUserStateParam)) || (lowSignalUser && query.params(EnableSlopFilterLowSignalUsers)) || query.params(EnableSlopFilter) ) } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val minFollowersThreshold = query.params(SlopMinFollowers) val (removed, kept) = candidates.partition { candidate => candidate.features.getOrElse(SlopAuthorFeature, false) && !candidate.features.getOrElse(InNetworkFeature, false) && candidate.features.getOrElse(AuthorFollowersFeature, None).forall(_ > minFollowersThreshold) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/TweetHydrationFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Filter out tweets that fail Tweetypie VF hydration */ object TweetHydrationFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("TweetHydration") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (kept, removed) = candidates.partition { candidate => candidate.features.getOrElse(IsHydratedFeature, false) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/WeeklyBookmarkFilter.scala ================================================ package com.twitter.home_mixer.functional_component.filter import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.BookmarkedTweetTimestamp import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.util.Time object WeeklyBookmarkFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("WeeklyBookmark") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (keptCandidates, removedCandidates) = candidates.partition { filterCandidate => filterCandidate.features .get(BookmarkedTweetTimestamp).exists { timestamp => val aWeekAgo = Time.now - 7.days Time.fromMilliseconds(timestamp) >= aWeekAgo } } Stitch.value( FilterResult( kept = keptCandidates.map(_.candidate), removed = removedCandidates.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/AllowForYouRecommendationsGate.scala ================================================ package com.twitter.home_mixer.functional_component.gate import com.twitter.home_mixer.model.HomeFeatures.ViewerAllowsForYouRecommendationsFeature import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * This gate disables out of network candidate pipelines when the AllowForYouRecommendations * user preference is set to false. * Defaults to true if the preference is not set. */ object AllowForYouRecommendationsGate extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("AllowForYouRecommendations") override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { val allowForYouRecommendations = query.features .flatMap(_.getOrElse(ViewerAllowsForYouRecommendationsFeature, Some(true))).getOrElse( true ) Stitch.value(allowForYouRecommendations) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module:test-user-mapper", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BookmarksTimeGate.scala ================================================ package com.twitter.home_mixer.functional_component.gate import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableBookmarksModuleWeekendGate import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import java.time.DayOfWeek import java.time.ZonedDateTime import javax.inject.Inject import javax.inject.Singleton @Singleton class BookmarksTimeGate @Inject() (serviceIdentifier: ServiceIdentifier) extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("BookmarksTime") // Serve on the weekends, with 48 hours injection time since persistence store is only 2 days. This way we have 2 // chances to show the module both on Saturday and Sunday private def isWeekend(zonedDateTime: ZonedDateTime) = { zonedDateTime.getDayOfWeek match { case DayOfWeek.SATURDAY => true case DayOfWeek.SUNDAY => true case _ => false } } override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { val gateDisabled = !query.params(EnableBookmarksModuleWeekendGate) val isDevel = serviceIdentifier.environment.toLowerCase != "prod" val isDarkRequest = query.features.flatMap { _.get(HasDarkRequestFeature) }.getOrElse(false) Stitch.value( gateDisabled || isDarkRequest || isDevel || isWeekend(query.queryTime.toZonedDateTime)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.scala ================================================ package com.twitter.home_mixer.functional_component.gate import com.twitter.conversions.DurationOps._ import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelinemixer.clients.manhattan.DismissInfo import com.twitter.timelineservice.suggests.thriftscala.SuggestType import com.twitter.util.Duration object DismissFatigueGate { // how long a dismiss action from user needs to be respected val DefaultBaseDismissDuration = 7.days val MaximumDismissalCountMultiplier = 4 } case class DismissFatigueGate( suggestType: SuggestType, dismissInfoFeature: Feature[PipelineQuery, Map[SuggestType, Option[DismissInfo]]], baseDismissDuration: Duration = DismissFatigueGate.DefaultBaseDismissDuration, ) extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("DismissFatigue") override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { val dismissInfoMap = query.features.map( _.getOrElse(dismissInfoFeature, Map.empty[SuggestType, Option[DismissInfo]])) val isVisible = dismissInfoMap .flatMap(_.get(suggestType)) .flatMap(_.map { info => val currentDismissalDuration = query.queryTime.since(info.lastDismissed) val targetDismissalDuration = dismissDurationForCount(info.count, baseDismissDuration) currentDismissalDuration > targetDismissalDuration }).getOrElse(true) Stitch.value(isVisible) } private def dismissDurationForCount( dismissCount: Int, dismissDuration: Duration ): Duration = // limit to maximum dismissal duration dismissDuration * Math.min(dismissCount, DismissFatigueGate.MaximumDismissalCountMultiplier) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ExcludeSoftUserGate.scala ================================================ package com.twitter.home_mixer.functional_component.gate import com.twitter.gizmoduck.{thriftscala => t} import com.twitter.home_mixer.model.HomeFeatures.UserTypeFeature import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * A Soft User is a user who is in the gradual onboarding state. This gate can be * used to turn off certain functionality like ads for these users. */ object ExcludeSoftUserGate extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("ExcludeSoftUser") override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { val softUser = query.features .exists(_.getOrElse(UserTypeFeature, None).exists(_ == t.UserType.Soft)) Stitch.value(!softUser) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/ExcludeSyntheticUserGate.scala ================================================ package com.twitter.home_mixer.functional_component.gate import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * A Synthetic User is a user who is created and managed on behalf of XCC's * Loadtesting framework. This gate can be used to turn off certain functionality like ads for * these users. */ object ExcludeSyntheticUserGate extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("ExcludeSyntheticUser") private val STRESS_TEST_APP_ID: Long = 1L override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { val isSyntheticUser = query.clientContext.appId.contains(STRESS_TEST_APP_ID) Stitch.value(!isSyntheticUser) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/PersistenceStoreDurationValidationGate.scala ================================================ package com.twitter.home_mixer.functional_component.gate import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.product_mixer.core.functional_component.configapi.StaticParam import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelinemixer.injection.store.persistence.TimelinePersistenceUtils import com.twitter.timelines.configapi.Param import com.twitter.timelines.util.client_info.ClientPlatform import com.twitter.util.Duration import com.twitter.util.Time /** * For users who max out the persistence store, we may serve certain modules too frequently. * Use this gate to prevent that. * * Gate stops the request if the time since the oldest entry < input duration * (or if there aren't enough entries, which can also cause a small duration) * * @param minInjectionIntervalParam the desired minimum interval between injections */ case class PersistenceStoreDurationValidationGate( minDuration: Param[Duration] = StaticParam(48.hours)) extends Gate[PipelineQuery] with TimelinePersistenceUtils { override val identifier: GateIdentifier = GateIdentifier("PersistenceStoreDurationValidation") private val MinEntries = 1500 override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { val continue = query.features.map { featureMap => val timelineResponses = featureMap.getOrElse(PersistenceEntriesFeature, Seq.empty) val clientPlatform = ClientPlatform.fromQueryOptions( clientAppId = query.clientContext.appId, userAgent = query.clientContext.userAgent.flatMap(UserAgent.fromString) ) val sortedResponses = responseByClient(clientPlatform, timelineResponses) val entryCount = sortedResponses.flatMap(_.entries).size val oldestTime = sortedResponses.lastOption.map(_.servedTime).getOrElse(Time.Bottom) val duration = Time.now.since(oldestTime) duration > query.params(minDuration) || entryCount < MinEntries } Stitch.value(continue.getOrElse(true)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RateLimitGate.scala ================================================ package com.twitter.home_mixer.functional_component.gate import com.twitter.home_mixer.model.HomeFeatures.ViewerHasPremiumTier import com.twitter.home_mixer.model.HomeFeatures.ViewerIsRateLimited import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object RateLimitGate extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("RateLimit") override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { val isRateLimited = query.features.map(_.getOrElse(ViewerIsRateLimited, false)) val hasPremiumTier = query.features.map(_.getOrElse(ViewerHasPremiumTier, false)) Stitch.value(isRateLimited.contains(false) || hasPremiumTier.contains(true)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RateLimitNotGate.scala ================================================ package com.twitter.home_mixer.functional_component.gate import com.twitter.home_mixer.model.HomeFeatures.ViewerHasPremiumTier import com.twitter.home_mixer.model.HomeFeatures.ViewerIsRateLimited import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object RateLimitNotGate extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("RateLimitNot") override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { val isRateLimited = query.features.map(_.getOrElse(ViewerIsRateLimited, false)) val hasPremiumTier = query.features.map(_.getOrElse(ViewerHasPremiumTier, false)) Stitch.value(isRateLimited.contains(true) && hasPremiumTier.contains(false)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RecentlyServedByServedTypeGate.scala ================================================ package com.twitter.home_mixer.functional_component.gate import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelinemixer.injection.store.persistence.TimelinePersistenceUtils import com.twitter.timelines.configapi.Param import com.twitter.timelineservice.model.rich.EntityIdType import com.twitter.timelineservice.model.TweetScoreV1 import com.twitter.util.Duration import com.twitter.util.Time import com.twitter.home_mixer.{thriftscala => hmt} /** * Gate used to reduce the frequency of injections based on specific served types. * This gate checks if any tweets in the persistence store have a served type equal to the specified targetServedType. * Note that the actual interval between injections may be less than the specified minInjectionIntervalParam * if data is unavailable or missing. For example, being deleted by the persistence store via a TTL or similar mechanism. * * @param minInjectionIntervalParam the desired minimum interval between injections * @param targetServedType the served type to check for in persisted tweets */ case class RecentlyServedByServedTypeGate( minInjectionIntervalParam: Param[Duration], targetServedType: hmt.ServedType) extends Gate[PipelineQuery] with TimelinePersistenceUtils { override val identifier: GateIdentifier = GateIdentifier("RecentlyServedByServedType") override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = Stitch( query.queryTime.since(getLastInjectionTime(query)) > query.params(minInjectionIntervalParam)) private def getLastInjectionTime(query: PipelineQuery) = query.features .flatMap { featureMap => val timelineResponses = featureMap.getOrElse(PersistenceEntriesFeature, Seq.empty) val latestResponseWithTargetServedTypeEntry = timelineResponses.find { response => response.entries.exists { entry => entry.entityIdType == EntityIdType.Tweet && entry.itemIds.exists { itemIds => itemIds.exists { itemId => itemId.tweetScore.exists { case tweetScore: TweetScoreV1 => tweetScore.servedType.contains(targetServedType.originalName) case _ => false } } } } } latestResponseWithTargetServedTypeEntry.map(_.servedTime) }.getOrElse(Time.Bottom) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextGate.scala ================================================ package com.twitter.home_mixer.functional_component.gate import com.twitter.home_mixer.model.request.DeviceContext.RequestContext import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Gate that fetches the request context from the device context and * continues if the request context matches *any* of the specified ones. */ case class RequestContextGate(requestContexts: Seq[RequestContext.Value]) extends Gate[PipelineQuery with HasDeviceContext] { override val identifier: GateIdentifier = GateIdentifier("RequestContext") override def shouldContinue(query: PipelineQuery with HasDeviceContext): Stitch[Boolean] = Stitch.value( requestContexts.exists(query.deviceContext.flatMap(_.requestContextValue).contains)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/RequestContextNotGate.scala ================================================ package com.twitter.home_mixer.functional_component.gate import com.twitter.home_mixer.model.request.DeviceContext.RequestContext import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Gate that fetches the request context from the device context and * continues if the request context does not match any of the specified ones. * * If no input request context is specified, the gate continues */ case class RequestContextNotGate(requestContexts: Seq[RequestContext.Value]) extends Gate[PipelineQuery with HasDeviceContext] { override val identifier: GateIdentifier = GateIdentifier("RequestContextNot") override def shouldContinue(query: PipelineQuery with HasDeviceContext): Stitch[Boolean] = Stitch.value( !requestContexts.exists(query.deviceContext.flatMap(_.requestContextValue).contains)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/SupportedLanguagesGate.scala ================================================ package com.twitter.home_mixer.functional_component.gate import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object SupportedLanguagesGate extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("SupportedLanguages") // Production languages which have high translation coverage for strings used in Home Timeline. private val supportedLanguages: Set[String] = Set( "ar", // Arabic "ar-x-fm", // Arabic (Female) "bg", // Bulgarian "bn", // Bengali "ca", // Catalan "cs", // Czech "da", // Danish "de", // German "el", // Greek "en", // English "en-gb", // British English "en-ss", // English Screen shot "en-xx", // English Pseudo "es", // Spanish "eu", // Basque "fa", // Farsi (Persian) "fi", // Finnish "fil", // Filipino "fr", // French "ga", // Irish "gl", // Galician "gu", // Gujarati "he", // Hebrew "hi", // Hindi "hr", // Croatian "hu", // Hungarian "id", // Indonesian "it", // Italian "ja", // Japanese "kn", // Kannada "ko", // Korean "mr", // Marathi "msa", // Malay "nl", // Dutch "no", // Norwegian "pl", // Polish "pt", // Portuguese "ro", // Romanian "ru", // Russian "sk", // Slovak "sr", // Serbian "sv", // Swedish "ta", // Tamil "th", // Thai "tr", // Turkish "uk", // Ukrainian "ur", // Urdu "vi", // Vietnamese "zh-cn", // Simplified Chinese "zh-tw" // Traditional Chinese ) override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = Stitch.value(query.getLanguageCode.forall(supportedLanguages.contains)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TestUserProbabilisticGate.scala ================================================ package com.twitter.home_mixer.functional_component.gate import javax.inject.Inject import javax.inject.Singleton import com.twitter.product_mixer.component_library.module.TestUserMapper import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Generic gate used to assign a probability that the associated Component / Pipeline operates * on a Synthetic test user. * * @param testUserMapper the testUserMapper utility object that evaluates if this is a test user */ @Singleton class TestUserProbabilisticGate @Inject() (testUserMapper: TestUserMapper) extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("TestUserProbabilistic") private val TEST_USERS_GATE_PROBABILITY = 0.05 override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { if (!testUserMapper.isTestUser( query.clientContext) || math.random < TEST_USERS_GATE_PROBABILITY) { Stitch.True } else { Stitch.False } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/TimelinesPersistenceStoreLastInjectionGate.scala ================================================ package com.twitter.home_mixer.functional_component.gate import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelinemixer.injection.store.persistence.TimelinePersistenceUtils import com.twitter.timelines.configapi.Param import com.twitter.timelines.util.client_info.ClientPlatform import com.twitter.timelineservice.model.rich.EntityIdType import com.twitter.util.Duration import com.twitter.util.Time /** * Gate used to reduce the frequency of injections. Note that the actual interval between injections may be * less than the specified minInjectionIntervalParam if data is unavailable or missing. For example, being deleted by * the persistence store via a TTL or similar mechanism. * * @param minInjectionIntervalParam the desired minimum interval between injections * @param persistenceEntriesFeature the feature for retrieving persisted timeline responses */ case class TimelinesPersistenceStoreLastInjectionGate( minInjectionIntervalParam: Param[Duration], entityIdType: EntityIdType.Value) extends Gate[PipelineQuery] with TimelinePersistenceUtils { override val identifier: GateIdentifier = GateIdentifier("TimelinesPersistenceStoreLastInjection") override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = Stitch( query.queryTime.since(getLastInjectionTime(query)) > query.params(minInjectionIntervalParam)) private def getLastInjectionTime(query: PipelineQuery) = query.features .flatMap { featureMap => val timelineResponses = featureMap.getOrElse(PersistenceEntriesFeature, Seq.empty) val clientPlatform = ClientPlatform.fromQueryOptions( clientAppId = query.clientContext.appId, userAgent = query.clientContext.userAgent.flatMap(UserAgent.fromString) ) val sortedResponses = responseByClient(clientPlatform, timelineResponses) val latestResponseWithEntityIdTypeEntry = sortedResponses.find(_.entries.exists(_.entityIdType == entityIdType)) latestResponseWithEntityIdTypeEntry.map(_.servedTime) }.getOrElse(Time.Bottom) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "common-internal/analytics/twitter-client-user-agent-parser/src/main/scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", "timelineservice/common:model", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.scala ================================================ package com.twitter.home_mixer.functional_component.query_transformer import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelinemixer.clients.persistence.EntryWithItemIds import com.twitter.timelines.persistence.thriftscala.RequestType import com.twitter.timelines.util.client_info.ClientPlatform import com.twitter.timelineservice.model.rich.EntityIdType import com.twitter.util.Time object EditedTweetsCandidatePipelineQueryTransformer extends CandidatePipelineQueryTransformer[PipelineQuery, Seq[Long]] { override val identifier: TransformerIdentifier = TransformerIdentifier("EditedTweets") // The time window for which a tweet remains editable after creation. private val EditTimeWindow = 60.minutes override def transform(query: PipelineQuery): Seq[Long] = { val applicableCandidates = getApplicableCandidates(query) if (applicableCandidates.nonEmpty) { // Include the response corresponding with the Previous Timeline Load (PTL). // Any tweets in it could have become stale since being served. val previousTimelineLoadTime = applicableCandidates.head.servedTime // The time window for editing a tweet is 60 minutes, // so we ignore responses older than (PTL Time - 60 mins). val inWindowCandidates: Seq[PersistenceStoreEntry] = applicableCandidates .takeWhile(_.servedTime.until(previousTimelineLoadTime) < EditTimeWindow) // Exclude the tweet IDs for which ReplaceEntry instructions have already been sent. val (tweetsAlreadyReplaced, tweetsToCheck) = inWindowCandidates .partition(_.entryWithItemIds.itemIds.exists(_.head.entryIdToReplace.nonEmpty)) val tweetIdFromEntry: PartialFunction[PersistenceStoreEntry, Long] = { case entry if entry.tweetId.nonEmpty => entry.tweetId.get } val tweetIdsAlreadyReplaced: Set[Long] = tweetsAlreadyReplaced.collect(tweetIdFromEntry).toSet val tweetIdsToCheck: Seq[Long] = tweetsToCheck.collect(tweetIdFromEntry) tweetIdsToCheck.filterNot(tweetIdsAlreadyReplaced.contains).distinct } else Seq.empty } // The candidates here come from the Timelines Persistence Store, via a query feature private def getApplicableCandidates(query: PipelineQuery): Seq[PersistenceStoreEntry] = { val userAgent = UserAgent.fromString(query.clientContext.userAgent.getOrElse("")) val clientPlatform = ClientPlatform.fromQueryOptions(query.clientContext.appId, userAgent) val sortedResponses = query.features .getOrElse(FeatureMap.empty) .getOrElse(PersistenceEntriesFeature, Seq.empty) .filter(_.clientPlatform == clientPlatform) .sortBy(-_.servedTime.inMilliseconds) val recentResponses = sortedResponses.indexWhere(_.requestType == RequestType.Initial) match { case -1 => sortedResponses case lastGetInitialIndex => sortedResponses.take(lastGetInitialIndex + 1) } recentResponses.flatMap { r => r.entries.collect { case entry if entry.entityIdType == EntityIdType.Tweet => PersistenceStoreEntry(entry, r.servedTime, r.clientPlatform, r.requestType) } }.distinct } } case class PersistenceStoreEntry( entryWithItemIds: EntryWithItemIds, servedTime: Time, clientPlatform: ClientPlatform, requestType: RequestType) { // Timelines Persistence Store currently includes 1 tweet ID per entryWithItemIds for tweets val tweetId: Option[Long] = entryWithItemIds.itemIds.flatMap(_.head.tweetId) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ "3rdparty/jvm/io/grpc:grpc-protobuf", "3rdparty/jvm/io/grpc:grpc-stub", "finagle-internal/finagle-grpc/src/main/scala", "finatra/inject/inject-utils/src/main/scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module:test-user-mapper", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "timelines/src/main/scala/com/twitter/timelines/clients/predictionservice", "timelineservice/common:model", "user_history_transformer/service/src/main/java/com/x/user_action_sequence", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala ================================================ package com.twitter.home_mixer.functional_component.scorer import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.common.{thriftscala => tl} import com.twitter.timelineservice.model.FeedbackEntry import com.twitter.timelineservice.{thriftscala => tls} import com.twitter.util.Time import scala.collection.mutable object FeedbackFatigueScorer extends Scorer[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] { override val identifier: ScorerIdentifier = ScorerIdentifier("FeedbackFatigue") override def features: Set[Feature[_, _]] = Set(ScoreFeature) override def onlyIf(query: PipelineQuery): Boolean = query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty) val DurationForDiscounting = 140.days private val ScoreMultiplierLowerBound = 0.2 private val ScoreMultiplierUpperBound = 1.0 private val ScoreMultiplierIncrementsCount = 4 private val ScoreMultiplierIncrement = (ScoreMultiplierUpperBound - ScoreMultiplierLowerBound) / ScoreMultiplierIncrementsCount private val ScoreMultiplierIncrementDurationInDays = DurationForDiscounting.inDays / ScoreMultiplierIncrementsCount.toDouble override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { val feedbackEntriesByEngagementType = query.features .getOrElse(FeatureMap.empty).getOrElse(FeedbackHistoryFeature, Seq.empty) .filter { entry => val timeSinceFeedback = query.queryTime.minus(entry.timestamp) timeSinceFeedback < DurationForDiscounting && entry.feedbackType == tls.FeedbackType.SeeFewer }.groupBy(_.engagementType) val authorsToDiscount = getUserDiscounts( query.queryTime, feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Tweet, Seq.empty)) val likersToDiscount = getUserDiscounts( query.queryTime, feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Like, Seq.empty)) val followersToDiscount = getUserDiscounts( query.queryTime, feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Follow, Seq.empty)) val retweetersToDiscount = getUserDiscounts( query.queryTime, feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty)) val featureMaps = candidates.map { candidate => val multiplier = getScoreMultiplier( candidate, authorsToDiscount, likersToDiscount, followersToDiscount, retweetersToDiscount ) val score = candidate.features.getOrElse(ScoreFeature, None) FeatureMapBuilder().add(ScoreFeature, score.map(_ * multiplier)).build() } Stitch.value(featureMaps) } def getScoreMultiplier( candidate: CandidateWithFeatures[TweetCandidate], authorsToDiscount: Map[Long, Double], likersToDiscount: Map[Long, Double], followersToDiscount: Map[Long, Double], retweetersToDiscount: Map[Long, Double], ): Double = { val originalAuthorId = CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L) val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0) val likers = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) val likerMultipliers = likers.flatMap(likersToDiscount.get) val likerMultiplier = if (likerMultipliers.nonEmpty && likers.size == likerMultipliers.size) likerMultipliers.max else 1.0 val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty) val followerMultipliers = followers.flatMap(followersToDiscount.get) val followerMultiplier = if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size && likers.isEmpty) followerMultipliers.max else 1.0 val authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L) val retweeterMultiplier = if (candidate.features.getOrElse(IsRetweetFeature, false)) retweetersToDiscount.getOrElse(authorId, 1.0) else 1.0 originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier } def getUserDiscounts( queryTime: Time, feedbackEntries: Seq[FeedbackEntry], ): Map[Long, Double] = { val userDiscounts = mutable.Map[Long, Double]() feedbackEntries .collect { case FeedbackEntry(_, _, tl.FeedbackEntity.UserId(userId), timestamp, _, _) => val timeSinceFeedback = queryTime.minus(timestamp) val multiplier = ((timeSinceFeedback.inDays / ScoreMultiplierIncrementDurationInDays) * ScoreMultiplierIncrement + ScoreMultiplierLowerBound) userDiscounts.update(userId, multiplier) } userDiscounts.toMap } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/NaviModelScorer.scala ================================================ package com.twitter.home_mixer.functional_component.scorer import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.NaviClientConfigFeature import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature import com.twitter.home_mixer.model.PredictedScoreFeature.PredictedScoreFeatureSet import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelIdParam import com.twitter.home_mixer.util.NaviScorerStatsHandler import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.module.TestUserMapper import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.feature.featuremap.datarecord.AllFeatures import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordExtractor import com.twitter.product_mixer.core.feature.featuremap.datarecord.FeatureMapSanitizer import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.util.Future import com.twitter.util.Return import java.util.UUID import javax.inject.Inject import javax.inject.Singleton import com.twitter.product_mixer.core.feature.datarecord.BaseDataRecordFeature object CommonFeaturesDataRecordFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object CandidateFeaturesDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton case class NaviModelScorer @Inject() ( predictClientFactory: PredictClientFactory, testUserMapper: TestUserMapper, statsReceiver: StatsReceiver) extends Scorer[PipelineQuery, TweetCandidate] { override val identifier: ScorerIdentifier = ScorerIdentifier("NaviModel") override val features: Set[Feature[_, _]] = Set( CommonFeaturesDataRecordFeature, CandidateFeaturesDataRecordFeature, PredictionRequestIdFeature, ) ++ PredictedScoreFeatureSet.asInstanceOf[Set[Feature[_, _]]] private val queryDataRecordAdapter = new DataRecordConverter(AllFeatures()) private val candidatesDataRecordAdapter = new DataRecordConverter(AllFeatures()) private val resultDataRecordExtractor = new DataRecordExtractor(PredictedScoreFeatureSet) private val modelStatsHandler = new NaviScorerStatsHandler(statsReceiver, getClass.getSimpleName) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { val modelId = query.params(ModelIdParam) val naviClientConfig = query.features.map(_.get(NaviClientConfigFeature)).get // Should always be present val modelStats = modelStatsHandler.getModelStats(query) val modelClient = predictClientFactory.getClient( naviClientConfig.clientName, naviClientConfig.customizedBatchSize) val predictionRequestId = UUID.randomUUID.getMostSignificantBits val candidateAdapter = candidatesDataRecordAdapter.toDataRecord(_) val commonRecord = query.features.map(queryDataRecordAdapter.toDataRecord) def getScores( candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Future[Seq[FeatureMap]] = { val features = candidates.map(_.features) val records = features.map(candidateAdapter) val responsesFut = modelClient.getPredictionsForBatch(records, commonRecord, modelId = Some(modelId)) responsesFut.map { responses => modelStats.failuresStat.add(responses.count(_.isThrow)) modelStats.responsesStat.add(responses.size) if (responses.size == candidates.size) { val predictedScoreFeatureMaps = responses.map { case Return(dataRecord) => resultDataRecordExtractor.fromDataRecord(dataRecord) case _ => resultDataRecordExtractor.fromDataRecord(new DataRecord()) } val featureMapSanitizer = new FeatureMapSanitizer[BaseDataRecordFeature[_, _]]( includeFeatures = PredictedScoreFeatureSet.toSet, // Ensure this is a Set[DRFeature] statsReceiver = statsReceiver ) // Sanitize the FeatureMaps val sanitizedFeatureMaps = featureMapSanitizer.sanitize(predictedScoreFeatureMaps) // Add Data Record to candidate Feature Map for logging in later stages sanitizedFeatureMaps.zip(records).map { case (predictedScoreFeatureMap, candidateRecord) => predictedScoreFeatureMap ++ FeatureMapBuilder() .add(CandidateFeaturesDataRecordFeature, candidateRecord) .add(CommonFeaturesDataRecordFeature, commonRecord.getOrElse(new DataRecord())) .add(PredictionRequestIdFeature, Some(predictionRequestId)) .build() } } else { modelStats.invalidResponsesCounter.incr() throw PipelineFailure(IllegalStateFailure, "Result size mismatched candidates size") } } } val scores = OffloadFuturePools.offloadBatchSeqToFutureSeq( candidates, getScores(_), naviClientConfig.customizedBatchSize.getOrElse(predictClientFactory.DefaultRequestBatchSize), offload = true ) Stitch.callFuture(scores) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/OONTweetScalingScorer.scala ================================================ package com.twitter.home_mixer.functional_component.scorer import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Scales scores of each out-of-network tweet by the specified scale factor */ object OONTweetScalingScorer extends Scorer[PipelineQuery, TweetCandidate] { override val identifier: ScorerIdentifier = ScorerIdentifier("OONTweetScaling") override val features: Set[Feature[_, _]] = Set(ScoreFeature) private val ScaleFactor = 0.75 override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { Stitch.value { candidates.map { candidate => val score = candidate.features.getOrElse(ScoreFeature, None) val updatedScore = if (selector(candidate)) score.map(_ * ScaleFactor) else score FeatureMapBuilder().add(ScoreFeature, updatedScore).build() } } } /** * We should only be applying this multiplier to Out-Of-Network tweets. * In-Network Retweets of Out-Of-Network tweets should not have this multiplier applied */ private def selector(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = { !candidate.features.getOrElse(InNetworkFeature, false) && !candidate.features.getOrElse(IsRetweetFeature, false) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/PhoenixModelRerankingScorer.scala ================================================ package com.twitter.home_mixer.functional_component.scorer import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature import com.twitter.home_mixer.model.HomeFeatures.PhoenixScoreFeature import com.twitter.home_mixer.model.PhoenixPredictedScoreFeature.PhoenixPredictedScoreFeatures import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixInferenceClusterParam import com.twitter.home_mixer.util.PhoenixScorerStatsHandler import com.twitter.home_mixer.util.RerankerUtil._ import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton case class PhoenixModelRerankingScorer @Inject() (statsReceiver: StatsReceiver) extends Scorer[PipelineQuery, TweetCandidate] { override val identifier: ScorerIdentifier = ScorerIdentifier("PhoenixModelReranking") override val features: Set[Feature[_, _]] = Set( PhoenixScoreFeature, // WeightedModelScoreFeature, remove temporarily to avoid overwriting navi weighted score DebugStringFeature ) private val StatsReadabilityMultiplier = 1000 private val modelStatsHandler = new PhoenixScorerStatsHandler(statsReceiver, getClass.getSimpleName) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { val cluster = query.params(PhoenixInferenceClusterParam).toString val modelStats = modelStatsHandler.getModelStats(query, cluster) val scoresAndWeightsSeq = candidates.map { candidate => PhoenixPredictedScoreFeatures.map { feature => val predictions = feature.extractScore(candidate.features, query) modelStats.trackPredictedScoreStats(feature, predictions) val isEligible = feature.isEligible(candidate.features) val score = if (predictions.nonEmpty && isEligible) predictions.max else 0.0 val weight = query.params(feature.modelWeightParam) (score, weight) } } val transformedScoresAndWeightsSeq = getScoresWithPerHeadMax(scoresAndWeightsSeq) val debugStrings: Seq[String] = candidates.map(_.features.getOrElse(DebugStringFeature, None).getOrElse("")) val featureMaps = transformedScoresAndWeightsSeq .zip(debugStrings) .map { case (transformedScores, debugStr) => val finalScore = aggregateWeightedScores(query, transformedScores, modelStats.negativeFilterCounter) val featureNames = PhoenixPredictedScoreFeatures.map(_.featureName) modelStats.scoreStat.add((finalScore * StatsReadabilityMultiplier).toFloat) val updatedDebugStr = computeDebugMetadata(debugStr, featureNames, transformedScores, finalScore) FeatureMapBuilder() .add(PhoenixScoreFeature, Some(finalScore)) .add(DebugStringFeature, Some(updatedDebugStr)) .build() } Stitch.value(featureMaps) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/PhoenixScorer.scala ================================================ package com.twitter.home_mixer.functional_component.scorer import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.UserActionsFeature import com.twitter.home_mixer.model.PhoenixPredictedScoreFeature.PhoenixPredictedScoreFeatureSet import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixInferenceClusterParam import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixTimeoutInMsParam import com.twitter.home_mixer.util.PhoenixUtils.createCandidateSets import com.twitter.home_mixer.util.PhoenixUtils.getPredictionResponseMap import com.twitter.home_mixer.util.PhoenixUtils.getTweetInfoFromCandidates import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.util.MemoizingStatsReceiver import com.twitter.stitch.Stitch import io.grpc.ManagedChannel import javax.inject.Inject import javax.inject.Singleton @Singleton case class PhoenixScorer @Inject() ( @Named("PhoenixClient") channelsMap: Map[PhoenixCluster.Value, Seq[ManagedChannel]], statsReceiver: StatsReceiver) extends Scorer[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] { override val identifier: ScorerIdentifier = ScorerIdentifier("Phoenix") override val features: Set[Feature[_, _]] = PhoenixPredictedScoreFeatureSet.asInstanceOf[Set[Feature[_, _]]] val memoizingStatsReceiver: MemoizingStatsReceiver = new MemoizingStatsReceiver( statsReceiver.scope(this.getClass.getSimpleName)) override def onlyIf(query: PipelineQuery): Boolean = { query.features.flatMap(_.getOrElse(UserActionsFeature, None)).isDefined } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch { val phoenixCluster = query.params(PhoenixInferenceClusterParam) val channels = channelsMap(phoenixCluster) val tweetInfos = getTweetInfoFromCandidates(candidates.map(_.candidate), candidates.map(_.features)) val request = createCandidateSets(query, tweetInfos) val timeoutMs = query.params(PhoenixTimeoutInMsParam) val predictionsMapStitch = getPredictionResponseMap( request, channels, phoenixCluster.toString, timeoutMs, memoizingStatsReceiver) predictionsMapStitch.map { predictionsMap => candidates.map { candidate => val sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None) match { case Some(sourceTweetId) => sourceTweetId case _ => candidate.candidate.id } val actionPredictionsMap = predictionsMap.getOrElse(sourceTweetId, Map.empty) val fmBuilder = FeatureMapBuilder() PhoenixPredictedScoreFeatureSet.map { feature => val predictions = feature.actions.flatMap(actionPredictionsMap.get) val score = if (predictions.nonEmpty) Some(predictions.max) else None fmBuilder.add(feature, score) } fmBuilder.build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/PredictClientFactory.scala ================================================ package com.twitter.home_mixer.functional_component.scorer import com.twitter.finagle.stats.BroadcastStatsReceiver import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecap import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapGPU import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapRealtime import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapSecondary import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapVideo import com.twitter.timelines.clients.predictionservice.PredictionGRPCService import com.twitter.timelines.clients.predictionservice.PredictionServiceGRPCClient import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton case class PredictClientFactory @Inject() ( @Named(NaviModelClientHomeRecap) homeRecapPredictionGRPCService: PredictionGRPCService, @Named(NaviModelClientHomeRecapSecondary) homeRecapSecondaryPredictionGRPCService: PredictionGRPCService, @Named(NaviModelClientHomeRecapRealtime) homeRecapRealtimePredictionGRPCService: PredictionGRPCService, @Named(NaviModelClientHomeRecapGPU) homeRecapGPUPredictionGRPCService: PredictionGRPCService, @Named(NaviModelClientHomeRecapVideo) homeRecapVideoPredictionGRPCService: PredictionGRPCService, statsReceiver: StatsReceiver) { val DefaultRequestBatchSize = 32 def getClient( clientName: String, customizedBatchSize: Option[Int] ): PredictionServiceGRPCClient = { clientName match { case NaviModelClientHomeRecap => homeRecapModelClient case NaviModelClientHomeRecapSecondary => homeRecapSecondaryModelClient case NaviModelClientHomeRecapRealtime => homeRecapRealtimeModelClient case NaviModelClientHomeRecapVideo => homeRecapVideoModelClient case NaviModelClientHomeRecapGPU => val gpuBatchSize = customizedBatchSize.getOrElse(DefaultRequestBatchSize) new PredictionServiceGRPCClient( service = homeRecapGPUPredictionGRPCService, statsReceiver = BroadcastStatsReceiver( Seq(statsReceiver, statsReceiver.scope("home_recap_gpu"))), requestBatchSize = gpuBatchSize, useCompact = false ) case _ => throw new IllegalArgumentException(s"Unknown clientName: $clientName") } } private lazy val homeRecapModelClient = new PredictionServiceGRPCClient( service = homeRecapPredictionGRPCService, statsReceiver = BroadcastStatsReceiver(Seq(statsReceiver, statsReceiver.scope("home_recap"))), requestBatchSize = DefaultRequestBatchSize, useCompact = false ) private lazy val homeRecapSecondaryModelClient = new PredictionServiceGRPCClient( service = homeRecapSecondaryPredictionGRPCService, statsReceiver = BroadcastStatsReceiver(Seq(statsReceiver, statsReceiver.scope("home_recap"))), requestBatchSize = DefaultRequestBatchSize, useCompact = false ) private lazy val homeRecapRealtimeModelClient = new PredictionServiceGRPCClient( service = homeRecapRealtimePredictionGRPCService, statsReceiver = BroadcastStatsReceiver( Seq(statsReceiver, statsReceiver.scope("home_recap_realtime"))), requestBatchSize = DefaultRequestBatchSize, useCompact = false ) private lazy val homeRecapVideoModelClient = new PredictionServiceGRPCClient( service = homeRecapVideoPredictionGRPCService, statsReceiver = BroadcastStatsReceiver( Seq(statsReceiver, statsReceiver.scope("home_recap_video"))), requestBatchSize = DefaultRequestBatchSize, useCompact = false ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/WeighedModelRerankingScorer.scala ================================================ package com.twitter.home_mixer.functional_component.scorer import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature import com.twitter.home_mixer.model.PredictedScoreFeature.PredictedScoreFeatures import com.twitter.home_mixer.util.NaviScorerStatsHandler import com.twitter.home_mixer.util.RerankerUtil._ import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton case class WeighedModelRerankingScorer @Inject() ( statsReceiver: StatsReceiver) extends Scorer[PipelineQuery, TweetCandidate] { override val identifier: ScorerIdentifier = ScorerIdentifier("WeightedModelReranking") override val features: Set[Feature[_, _]] = Set( ScoreFeature, WeightedModelScoreFeature, DebugStringFeature ) private val modelStatsHandler = new NaviScorerStatsHandler(statsReceiver, getClass.getSimpleName) private val StatsReadabilityMultiplier = 1000 override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { val modelStats = modelStatsHandler.getModelStats(query) val scoresAndWeightsSeq = candidates.map(computeModelScores(query, _, Some(modelStats))) val transformedScoresAndWeightsSeq = getScoresWithPerHeadMax(scoresAndWeightsSeq) val debugStrings: Seq[String] = candidates.map(_.features.getOrElse(DebugStringFeature, None).getOrElse("")) val featureMaps = transformedScoresAndWeightsSeq .zip(debugStrings) .map { case (transformedScores, debugStr) => val finalScore = aggregateWeightedScores(query, transformedScores, modelStats.negativeFilterCounter) val featureNames = PredictedScoreFeatures.map(_.statName) modelStats.scoreStat.add((finalScore * StatsReadabilityMultiplier).toFloat) val updatedDebugStr = computeDebugMetadata(debugStr, featureNames, transformedScores, finalScore) FeatureMapBuilder() .add(ScoreFeature, Some(finalScore)) .add(WeightedModelScoreFeature, Some(finalScore)) .add(DebugStringFeature, Some(updatedDebugStr)) .build() } Stitch.value(featureMaps) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", "src/scala/com/twitter/suggests/controller_data", "stringcenter/client", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/DebunchCandidates.scala ================================================ package com.twitter.home_mixer.functional_component.selector import com.twitter.home_mixer.functional_component.selector.DebunchCandidates.TrailingTweetsMinSize import com.twitter.home_mixer.functional_component.selector.DebunchCandidates.TrailingTweetsPortionToKeep import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.functional_component.common.CandidateScope.PartitionedCandidates import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.selector.SelectorResult import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery trait MustDebunch { def apply(candidate: CandidateWithDetails): Boolean } object DebunchCandidates { val TrailingTweetsMinSize = 5 val TrailingTweetsPortionToKeep = 0.1 } /** * This selector rearranges the candidates to only allow bunches of size [[maxBunchSize]], where a * bunch is a consecutive sequence of candidates that meet [[mustDebunch]]. */ case class DebunchCandidates( override val pipelineScope: CandidateScope, mustDebunch: MustDebunch, maxBunchSize: Int) extends Selector[PipelineQuery] { override def apply( query: PipelineQuery, remainingCandidates: Seq[CandidateWithDetails], result: Seq[CandidateWithDetails] ): SelectorResult = { val PartitionedCandidates(selectedCandidates, otherCandidates) = pipelineScope.partition(remainingCandidates) val mutableCandidates = collection.mutable.ListBuffer(selectedCandidates: _*) var candidatePointer = 0 var nonDebunchPointer = 0 var bunchSize = 0 var finalNonDebunch = -1 while (candidatePointer < mutableCandidates.size) { if (mustDebunch(mutableCandidates(candidatePointer))) bunchSize += 1 else { bunchSize = 0 finalNonDebunch = candidatePointer } if (bunchSize > maxBunchSize) { nonDebunchPointer = Math.max(candidatePointer, nonDebunchPointer) while (nonDebunchPointer < mutableCandidates.size && mustDebunch(mutableCandidates(nonDebunchPointer))) { nonDebunchPointer += 1 } if (nonDebunchPointer == mutableCandidates.size) candidatePointer = mutableCandidates.size else { val nextNonDebunch = mutableCandidates(nonDebunchPointer) mutableCandidates.remove(nonDebunchPointer) mutableCandidates.insert(candidatePointer, nextNonDebunch) bunchSize = 0 finalNonDebunch = candidatePointer } } candidatePointer += 1 } val debunchedCandidates = if (query.features.exists(_.getOrElse(GetNewerFeature, false))) { val trailingTweetsSize = mutableCandidates.size - finalNonDebunch - 1 val keepCandidates = finalNonDebunch + 1 + Math.max(TrailingTweetsMinSize, TrailingTweetsPortionToKeep * trailingTweetsSize).toInt mutableCandidates.toList.take(keepCandidates) } else mutableCandidates.toList val updatedCandidates = otherCandidates ++ debunchedCandidates SelectorResult(remainingCandidates = updatedCandidates, result = result) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/RandomShuffleCandidates.scala ================================================ package com.twitter.home_mixer.functional_component.selector import com.twitter.product_mixer.core.functional_component.common.AllPipelines import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.selector.SelectorResult import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery import scala.util.Random object RandomShuffleCandidates extends Selector[PipelineQuery] { override def apply( query: PipelineQuery, remainingCandidates: Seq[CandidateWithDetails], result: Seq[CandidateWithDetails] ): SelectorResult = { val shuffledRemainingCandidates = Random.shuffle(remainingCandidates) SelectorResult(remainingCandidates = shuffledRemainingCandidates, result = result) } override def pipelineScope: CandidateScope = AllPipelines } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/ScoreAveragingPositionSelector.scala ================================================ package com.twitter.home_mixer.functional_component.selector import com.twitter.home_mixer.model.HomeFeatures.IsBoostedCandidateFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.UserFollowersCountFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CategoryColdStartProbabilisticReturnParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CategoryColdStartTierOneProbabilityParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ContentExplorationBoostPosParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ContentExplorationViewerMaxFollowersParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.DeepRetrievalBoostPosParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.DeepRetrievalI2iProbabilityParam import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.common.AllPipelines import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.selector.SelectorResult import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery import scala.util.Random /** * Abstract base class specifically for fixed position selectors that need score averaging * Contains complete selector logic, supports removing target candidates from remaining candidates * and assigning them the average score of adjacent candidates */ abstract class ScoreAveragingPositionSelector extends Selector[PipelineQuery] { // Abstract methods that subclasses need to implement def getOffset(query: PipelineQuery): Int def getNumCandidatesToBoost(query: PipelineQuery): Int def selectEligibleCandidates(query: PipelineQuery, candidate: CandidateWithDetails): Boolean override def apply( query: PipelineQuery, remainingCandidates: Seq[CandidateWithDetails], result: Seq[CandidateWithDetails] ): SelectorResult = { val eligibleCandidates = remainingCandidates.filter(c => selectEligibleCandidates(query, c)) val targetCandidates = eligibleCandidates.take(getNumCandidatesToBoost(query)) if (targetCandidates.nonEmpty) { val filteredRemainingCandidates = remainingCandidates.diff(targetCandidates) val offset = getOffset(query) // Get scores from adjacent positions for average calculation val adjacentScores = getAdjacentScores(filteredRemainingCandidates, offset) // Assign target candidates scores based on the average of adjacent candidates val scoredTargetCandidates = targetCandidates.map(candidate => assignAverageScore(candidate, adjacentScores)) val updatedRemainingCandidates = if (offset >= 0 && offset <= filteredRemainingCandidates.length) { filteredRemainingCandidates.slice(0, offset) ++ scoredTargetCandidates ++ filteredRemainingCandidates.slice(offset, filteredRemainingCandidates.length) } else { filteredRemainingCandidates ++ scoredTargetCandidates } SelectorResult(remainingCandidates = updatedRemainingCandidates, result = result) } else { SelectorResult(remainingCandidates = remainingCandidates, result = result) } } // Get adjacent position scores def getAdjacentScores( filteredRemainingCandidates: Seq[CandidateWithDetails], offset: Int ): Seq[Option[Double]] = { if (offset >= 1 && offset < filteredRemainingCandidates.length) { Seq( filteredRemainingCandidates(offset - 1).features.getOrElse(ScoreFeature, None), filteredRemainingCandidates(offset).features.getOrElse(ScoreFeature, None) ) } else if (offset == 0 && filteredRemainingCandidates.nonEmpty) { Seq(filteredRemainingCandidates.head.features.getOrElse(ScoreFeature, None)) } else if (offset >= filteredRemainingCandidates.length && filteredRemainingCandidates.nonEmpty) { Seq(filteredRemainingCandidates.last.features.getOrElse(ScoreFeature, None)) } else { Seq.empty } } // Assign scores to candidates based on the average score of adjacent candidates def assignAverageScore( candidate: CandidateWithDetails, adjacentScores: Seq[Option[Double]] ): CandidateWithDetails = { val validScores = adjacentScores.flatten val avgScore = if (validScores.nonEmpty) { Some(validScores.sum / validScores.size) } else None candidate match { case item: ItemCandidateWithDetails if avgScore.isDefined => val updatedFeatures = FeatureMapBuilder() .add(ScoreFeature, avgScore) .add(IsBoostedCandidateFeature, true) .build() item.copy(features = item.features ++ updatedFeatures) case _ => candidate } } def pipelineScope: CandidateScope = AllPipelines } /** * Concrete implementations of score averaging position selectors */ object SortFixedPositionContentExplorationSimclusterColdPostsCandidates extends ScoreAveragingPositionSelector { override def getOffset(query: PipelineQuery): Int = query.params.getInt(ContentExplorationBoostPosParam) override def getNumCandidatesToBoost(query: PipelineQuery): Int = 1 override def selectEligibleCandidates( query: PipelineQuery, candidate: CandidateWithDetails ): Boolean = { val servedType = candidate.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined) servedType == hmt.ServedType.ForYouContentExplorationSimclusterColdPosts } } object SortFixedPositionContentExplorationMixedCandidates extends ScoreAveragingPositionSelector { override def getOffset(query: PipelineQuery): Int = query.params.getInt(ContentExplorationBoostPosParam) override def getNumCandidatesToBoost(query: PipelineQuery): Int = 1 override def selectEligibleCandidates( query: PipelineQuery, candidate: CandidateWithDetails ): Boolean = true override def apply( query: PipelineQuery, remainingCandidates: Seq[CandidateWithDetails], result: Seq[CandidateWithDetails] ): SelectorResult = { val chosenServedType = if (Random.nextDouble() < query.params.getDouble(CategoryColdStartTierOneProbabilityParam)) hmt.ServedType.ForYouContentExploration else hmt.ServedType.ForYouContentExplorationTier2 val eligibleCandidates = remainingCandidates .filter(_.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined) == chosenServedType) val viewerMaxFollowers = query.params(ContentExplorationViewerMaxFollowersParam) val viewerFollowers = query.features.flatMap(_.getOrElse(UserFollowersCountFeature, None)) val returnCandidatesRandomDraw = Random.nextDouble() < query.params.getDouble(CategoryColdStartProbabilisticReturnParam) if (viewerFollowers.forall(_ < viewerMaxFollowers) && eligibleCandidates.nonEmpty && returnCandidatesRandomDraw) { val targetCandidates = eligibleCandidates.take(getNumCandidatesToBoost(query)) val filteredRemainingCandidates = remainingCandidates.diff(targetCandidates) val offset = getOffset(query) // Get scores from adjacent positions for average calculation val adjacentScores = getAdjacentScores(filteredRemainingCandidates, offset) // Assign target candidates scores based on the average of adjacent candidates val scoredTargetCandidates = targetCandidates.map(candidate => assignAverageScore(candidate, adjacentScores)) val updatedRemainingCandidates = if (offset >= 0 && offset <= filteredRemainingCandidates.length) { filteredRemainingCandidates.slice(0, offset) ++ scoredTargetCandidates ++ filteredRemainingCandidates.slice(offset, filteredRemainingCandidates.length) } else filteredRemainingCandidates ++ scoredTargetCandidates SelectorResult(remainingCandidates = updatedRemainingCandidates, result = result) } else SelectorResult(remainingCandidates = remainingCandidates, result = result) } } object SortFixedPositionDeepRetrievalMixedCandidates extends ScoreAveragingPositionSelector { override def getOffset(query: PipelineQuery): Int = query.params.getInt(DeepRetrievalBoostPosParam) override def getNumCandidatesToBoost(query: PipelineQuery): Int = 1 override def selectEligibleCandidates( query: PipelineQuery, candidate: CandidateWithDetails ): Boolean = true override def apply( query: PipelineQuery, remainingCandidates: Seq[CandidateWithDetails], result: Seq[CandidateWithDetails] ): SelectorResult = { val chosenServedType = if (Random.nextDouble() < query.params.getDouble(DeepRetrievalI2iProbabilityParam)) hmt.ServedType.ForYouContentExplorationDeepRetrievalI2i else hmt.ServedType.ForYouContentExplorationTier2DeepRetrievalI2i val eligibleCandidates = remainingCandidates .filter(_.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined) == chosenServedType) val viewerMaxFollowers = query.params(ContentExplorationViewerMaxFollowersParam) val viewerFollowers = query.features.flatMap(_.getOrElse(UserFollowersCountFeature, None)) if (viewerFollowers.forall(_ < viewerMaxFollowers) && eligibleCandidates.nonEmpty) { val targetCandidates = Seq(eligibleCandidates(Random.nextInt(eligibleCandidates.size))) val filteredRemainingCandidates = remainingCandidates.diff(targetCandidates) val offset = getOffset(query) // Get scores from adjacent positions for average calculation val adjacentScores = getAdjacentScores(filteredRemainingCandidates, offset) // Assign target candidates scores based on the average of adjacent candidates val scoredTargetCandidates = targetCandidates.map(candidate => assignAverageScore(candidate, adjacentScores)) val updatedRemainingCandidates = if (offset >= 0 && offset <= filteredRemainingCandidates.length) { filteredRemainingCandidates.slice(0, offset) ++ scoredTargetCandidates ++ filteredRemainingCandidates.slice(offset, filteredRemainingCandidates.length) } else filteredRemainingCandidates ++ scoredTargetCandidates SelectorResult(remainingCandidates = updatedRemainingCandidates, result = result) } else SelectorResult(remainingCandidates = remainingCandidates, result = result) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/SortFixedPositionCandidates.scala ================================================ package com.twitter.home_mixer.functional_component.selector import com.twitter.product_mixer.core.functional_component.common.AllPipelines import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.selector.SelectorResult import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery abstract class SortFixedPositionCandidates extends Selector[PipelineQuery] { def getOffset(query: PipelineQuery): Int def getNumCandidatesToBoost(query: PipelineQuery): Int def selectEligibleCandidates( query: PipelineQuery, candidate: CandidateWithDetails ): Boolean def selectTargetFromRemainingCandidates( query: PipelineQuery, remainingCandidates: Seq[CandidateWithDetails] ): Seq[CandidateWithDetails] = { val numCandidatesToBoost = getNumCandidatesToBoost(query) val eligibleCandidates = remainingCandidates .filter(candidate => selectEligibleCandidates(query, candidate)) eligibleCandidates.take(numCandidatesToBoost) } override def apply( query: PipelineQuery, remainingCandidates: Seq[CandidateWithDetails], result: Seq[CandidateWithDetails] ): SelectorResult = { val targetCandidates = selectTargetFromRemainingCandidates(query, remainingCandidates) val offset = getOffset(query) if (targetCandidates.nonEmpty) { val updatedRemainingCandidates = if (offset >= 0 && offset < remainingCandidates.length) { remainingCandidates.slice(0, offset) ++ targetCandidates ++ remainingCandidates.slice( offset, remainingCandidates.length) } else { remainingCandidates ++ targetCandidates } SelectorResult( remainingCandidates = updatedRemainingCandidates.distinct, result = result ) } else { SelectorResult(remainingCandidates = remainingCandidates, result = result) } } override def pipelineScope: CandidateScope = AllPipelines } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateConversationModuleId.scala ================================================ package com.twitter.home_mixer.functional_component.selector import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.functional_component.common.CandidateScope.PartitionedCandidates import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.selector.SelectorResult import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery /** * This selector updates the id of the conversation modules to be the head of the module's id. */ case class UpdateConversationModuleId( override val pipelineScope: CandidateScope) extends Selector[PipelineQuery] { override def apply( query: PipelineQuery, remainingCandidates: Seq[CandidateWithDetails], result: Seq[CandidateWithDetails] ): SelectorResult = { val PartitionedCandidates(selectedCandidates, otherCandidates) = pipelineScope.partition(remainingCandidates) val updatedCandidates = selectedCandidates.map { case module @ ModuleCandidateWithDetails(candidates, presentationOpt, _) => val updatedPresentation = presentationOpt.map { case urtModule @ UrtModulePresentation(timelineModule) => urtModule.copy(timelineModule = timelineModule.copy(id = candidates.head.candidateIdLong)) } module.copy(presentation = updatedPresentation) case candidate => candidate } SelectorResult(remainingCandidates = updatedCandidates ++ otherCandidates, result = result) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateHomeClientEventDetails.scala ================================================ package com.twitter.home_mixer.functional_component.selector import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventDetailsBuilder import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModule2DisplayedTweetsFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleHasGapFeature import com.twitter.home_mixer.model.HomeFeatures.GrokCategoryDataRecordFeature import com.twitter.home_mixer.model.HomeFeatures.MaxSingleAuthorCountFeature import com.twitter.home_mixer.model.HomeFeatures.MaxSingleCategoryCountFeature import com.twitter.home_mixer.model.HomeFeatures.PositionFeature import com.twitter.home_mixer.model.HomeFeatures.ServedInConversationModuleFeature import com.twitter.home_mixer.model.HomeFeatures.ServedSizeFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.UniqueAuthorCountFeature import com.twitter.home_mixer.model.HomeFeatures.UniqueCategoryCountFeature import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.selector.SelectorResult import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem import com.twitter.product_mixer.core.pipeline.PipelineQuery /** * Builds serialized tweet type metrics controller data and updates Client Event Details * and Candidate Presentations with this info. * * Currently only updates presentation of Item Candidates. This needs to be updated * when modules are added. * * This is implemented as a Selector instead of a Decorator in the Candidate Pipeline * because we need to add controller data that looks at the final timeline as a whole * (e.g. served size, final candidate positions). * * @param candidatePipelines - only candidates from the specified pipeline will be updated */ case class UpdateHomeClientEventDetails(candidatePipelines: Set[CandidatePipelineIdentifier]) extends Selector[PipelineQuery] { override val pipelineScope: CandidateScope = SpecificPipelines(candidatePipelines) private val detailsBuilder = HomeClientEventDetailsBuilder() override def apply( query: PipelineQuery, remainingCandidates: Seq[CandidateWithDetails], result: Seq[CandidateWithDetails] ): SelectorResult = { val selectedCandidates = result.filter(pipelineScope.contains) val authorCounts: Map[Long, Int] = selectedCandidates .flatMap(_.features.getOrElse(AuthorIdFeature, None)) .groupBy(identity) .map { case (authorId, ids) => authorId -> ids.length } val categoryCounts: Map[String, Int] = selectedCandidates .flatMap(_.features.getOrElse(GrokCategoryDataRecordFeature, None).getOrElse(Map.empty).keys) .groupBy(identity) .map { case (category, categories) => category -> categories.length } val resultFeatures = FeatureMapBuilder() .add(ServedSizeFeature, Some(selectedCandidates.size)) .add(UniqueAuthorCountFeature, Some(authorCounts.size)) .add( MaxSingleAuthorCountFeature, Some(if (authorCounts.values.nonEmpty) authorCounts.values.max else 0)) .add(UniqueCategoryCountFeature, Some(categoryCounts.size)) .add( MaxSingleCategoryCountFeature, Some(if (categoryCounts.values.nonEmpty) categoryCounts.values.max else 0)) .build() val updatedResult = result.zipWithIndex.map { case (item @ ItemCandidateWithDetails(candidate, _, _), position) if pipelineScope.contains(item) => val resultCandidateFeatures = FeatureMapBuilder() .add(PositionFeature, Some(position)) .build() updateItemPresentation(query, item, resultFeatures, resultCandidateFeatures) case (module @ ModuleCandidateWithDetails(candidates, presentation, features), position) if pipelineScope.contains(module) => val resultCandidateFeatures = FeatureMapBuilder() .add(PositionFeature, Some(position)) .add(ServedInConversationModuleFeature, true) .add(ConversationModule2DisplayedTweetsFeature, module.candidates.size == 2) .add( ConversationModuleHasGapFeature, module.candidates.last.features.getOrElse(AncestorsFeature, Seq.empty).size > 2) .add(ServedTypeFeature, module.candidates.last.features.get(ServedTypeFeature)) .build() val updatedItemCandidates = candidates.map(updateItemPresentation(query, _, resultFeatures, resultCandidateFeatures)) val updatedCandidateFeatures = features ++ resultFeatures ++ resultCandidateFeatures val updatedPresentation = presentation.map { case urtModule @ UrtModulePresentation(timelineModule) => val clientEventDetails = detailsBuilder( query, candidates.last.candidate, query.features.get ++ updatedCandidateFeatures) val updatedClientEventInfo = timelineModule.clientEventInfo.map(_.copy(details = clientEventDetails)) val updatedTimelineModule = timelineModule.copy(clientEventInfo = updatedClientEventInfo) urtModule.copy(timelineModule = updatedTimelineModule) } module.copy( candidates = updatedItemCandidates, presentation = updatedPresentation, features = updatedCandidateFeatures ) case (any, position) => any } SelectorResult(remainingCandidates = remainingCandidates, result = updatedResult) } private def updateItemPresentation( query: PipelineQuery, item: ItemCandidateWithDetails, resultCandidateFeatures: FeatureMap, resultFeatures: FeatureMap, ): ItemCandidateWithDetails = { val updatedItemCandidateFeatures = item.features ++ resultFeatures ++ resultCandidateFeatures val updatedPresentation = item.presentation.map { case urtItem @ UrtItemPresentation(timelineItem: TweetItem, _) => val clientEventDetails = detailsBuilder(query, item.candidate, query.features.get ++ updatedItemCandidateFeatures) val updatedClientEventInfo = timelineItem.clientEventInfo.map(_.copy(details = clientEventDetails)) val updatedTimelineItem = timelineItem.copy(clientEventInfo = updatedClientEventInfo) urtItem.copy(timelineItem = updatedTimelineItem) case any => any } item.copy(presentation = updatedPresentation, features = updatedItemCandidateFeatures) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector/UpdateNewTweetsPillDecoration.scala ================================================ package com.twitter.home_mixer.functional_component.selector import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration.NumAvatars import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.param.HomeGlobalParams.EnableNewTweetsPillAvatarsParam import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.ShowAlertCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.selector.SelectorResult import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowAlert import com.twitter.product_mixer.core.model.marshalling.response.urt.richtext.RichText import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.core.ExternalString object UpdateNewTweetsPillDecoration { val NumAvatars = 3 } case class UpdateNewTweetsPillDecoration[Query <: PipelineQuery with HasDeviceContext]( override val pipelineScope: CandidateScope, stringCenter: StringCenter, seeNewTweetsString: ExternalString, tweetedString: ExternalString) extends Selector[Query] { override def apply( query: Query, remainingCandidates: Seq[CandidateWithDetails], result: Seq[CandidateWithDetails] ): SelectorResult = { val (alerts, otherCandidates) = remainingCandidates.partition(candidate => candidate.isCandidateType[ShowAlertCandidate]() && pipelineScope.contains(candidate)) val updatedCandidates = alerts .collectFirst { case newTweetsPill: ItemCandidateWithDetails => val userIds = CandidatesUtil .getItemCandidatesWithOnlyModuleLast(result) .filter(candidate => candidate.isCandidateType[TweetCandidate]() && pipelineScope.contains(candidate)) .filterNot(_.features.getOrElse(IsRetweetFeature, false)) .flatMap(_.features.getOrElse(AuthorIdFeature, None)) .filterNot(_ == query.getRequiredUserId) .distinct val updatedPresentation = newTweetsPill.presentation.map { case presentation: UrtItemPresentation => presentation.timelineItem match { case alert: ShowAlert => val text = if (useAvatars(query, userIds)) tweetedString else seeNewTweetsString val richText = RichText( text = stringCenter.prepare(text), entities = List.empty, rtl = None, alignment = None) val updatedAlert = alert.copy(userIds = Some(userIds.take(NumAvatars)), richText = Some(richText)) presentation.copy(timelineItem = updatedAlert) } } otherCandidates :+ newTweetsPill.copy(presentation = updatedPresentation) }.getOrElse(remainingCandidates) SelectorResult(remainingCandidates = updatedCandidates, result = result) } private def useAvatars(query: Query, userIds: Seq[Long]): Boolean = { val enableAvatars = query.params(EnableNewTweetsPillAvatarsParam) enableAvatars && userIds.size >= NumAvatars } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BUILD.bazel ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ "3rdparty/jvm/com/twitter/storehaus:core", "eventbus/client/src/main/scala/com/twitter/eventbus/client", "finagle/finagle-mysql/src/main/scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/light_ranking_features", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/non_ml_features", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweet_mixer", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/communities_to_join", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/job", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/recruiting_organization", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_subscribe_module", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "scribelib/marshallers/src/main/scala/com/twitter/scribelib/marshallers", "src/scala/com/twitter/suggests/controller_data", "src/scala/com/twitter/timelines/prediction/adapters/twistly", "src/scala/com/twitter/timelines/prediction/adapters/two_hop_features", "src/scala/com/twitter/timelines/prediction/features/large_embeddings", "src/thrift/com/twitter/timelines/impression_store:thrift-scala", "src/thrift/com/twitter/timelines/served_candidates_logging:served_candidates_logging-scala", "src/thrift/com/twitter/timelines/suggests/common:data_record_metadata-scala", "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", "strato/config/columns/videoRecommendations/twitterClip:twitterClip-strato-client", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", "timelines/ml:pldr-client", "timelines/ml:pldr-conversion", "timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding", "timelines/src/main/scala/com/twitter/timelines/clientconfig", "timelines/src/main/scala/com/twitter/timelines/util/stats", "timelineservice/common:model", "tweet-mixer/thrift/src/main/thrift:thrift-scala", "user_session_store/src/main/scala/com/twitter/user_session_store", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/BaseCacheCandidateFeaturesSideEffect.scala ================================================ package com.twitter.home_mixer.functional_component.side_effect import com.twitter.conversions.DurationOps._ import com.twitter.finagle.filter.OffloadFilter import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.mysql.Client import com.twitter.finagle.mysql.Transactions import com.twitter.finagle.offload.OffloadFuturePool import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.transport.Transport import com.twitter.finagle.util.DefaultTimer import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content.ClipEmbeddingFeaturesAdapter import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.content.TextTokensFeaturesAdapter import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.light_ranking_features.LightRankingCandidateFeatures import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.light_ranking_features.LightRankingCandidateFeaturesAdapter import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features.NonMLCandidateFeatures import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features.NonMLCandidateFeaturesAdapter import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features.NonMLCommonFeatures import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features.NonMLCommonFeaturesAdapter import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.twhin_embeddings.TwhinVideoEmbeddingsAdapter import com.twitter.home_mixer.functional_component.scorer.CandidateFeaturesDataRecordFeature import com.twitter.home_mixer.model.HomeFeatures.ClientIdFeature import com.twitter.home_mixer.model.HomeFeatures.GuestIdFeature import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextTokensFeature import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature import com.twitter.home_mixer.model.PredictedScoreFeature.PredictedScoreFeatureSet import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.ScoredTweetsProduct import com.twitter.home_mixer.model.request.ScoredVideoTweetsProduct import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableClipEmbeddingFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetTextTokensEmbeddingFeatureScribingParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetVideoAggregatedWatchTimeFeatureScribingParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinVideoFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableVideoClipEmbeddingFeatureHydrationDeciderParam import com.twitter.home_mixer.param.HomeGlobalParams.IsSelectedByHeavyRankerCountParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.util.CandidatesUtil.getOriginalAuthorId import com.twitter.home_mixer.util.CandidatesUtil.getOriginalTweetId import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecordMerger import com.twitter.ml.api.{thriftscala => ml} import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter import com.twitter.product_mixer.core.feature.featuremap.datarecord.SpecificFeatures import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateSourcePosition import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.HasMarshalling import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding import com.twitter.stitch.Stitch import com.twitter.storehaus.ReadableStore import com.twitter.storehaus.Store import com.twitter.strato.generated.client.videoRecommendations.twitterClip.TwitterClipEmbeddingMhClientColumn import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.CandidateAndCommonFeaturesStreamingUtils import com.twitter.timelines.ml.pldr.client.MysqlClientUtils import com.twitter.timelines.ml.pldr.client.VersionedMetadataCacheClient import com.twitter.timelines.ml.pldr.conversion.VersionIdAndFeatures import com.twitter.timelines.prediction.adapters.twistly.VideoAggregatedWatchTimeFeaturesAdapter import com.twitter.timelines.prediction.features.large_embeddings.LargeEmbeddingsFeatures.AllCandidateLargeEmbeddingsFeatures import com.twitter.timelines.served_candidates_logging.{thriftscala => sc} import com.twitter.timelines.suggests.common.data_record_metadata.{thriftscala => drmd} import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} import com.twitter.timelines.util.stats.FutureObserver import com.twitter.timelines.util.stats.OptionObserver import com.twitter.twistly.thriftscala.VideoViewEngagementType import com.twitter.twistly.thriftscala.WatchTimeMetadata import com.twitter.twistly.{thriftscala => ts} import com.twitter.util.Future import com.twitter.util.Try import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer @Singleton class BaseCacheCandidateFeaturesSideEffect @Inject() ( dataRecordMetadataStoreConfigsYml: String, store: Store[ sc.CandidateFeatureKey, pldr.PolyDataRecord ], tweetWatchTimeMetadataStore: ReadableStore[(Long, VideoViewEngagementType), WatchTimeMetadata], twitterClipEmbeddingMhClientColumn: TwitterClipEmbeddingMhClientColumn, twhinVideoStore: ReadableStore[Long, TwhinTweetEmbedding], statsReceiver: StatsReceiver) extends PipelineResultSideEffect[PipelineQuery, HasMarshalling] with Conditionally[PipelineQuery, HasMarshalling] with Logging { override def onlyIf( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: HasMarshalling ): Boolean = { val serviceIdentifier = ServiceIdentifier.fromCertificate(Transport.peerCertificate) (selectedCandidates.nonEmpty || remainingCandidates.nonEmpty || droppedCandidates.nonEmpty) && serviceIdentifier.role != "video-mixer" } override val alerts: Seq[SuccessRateAlert] = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(98.5)) val identifier: SideEffectIdentifier = SideEffectIdentifier("BaseCacheCandidateFeaturesSideEffect") val statScope: String = this.getClass.getSimpleName private val scopedStatsReceiver = statsReceiver.scope(statScope) private val metadataFetchFailedCounter = scopedStatsReceiver.counter("metadataFetchFailed") private val writesRequestCounter = scopedStatsReceiver.counter("writesRequests") private val writesFailedCounter = scopedStatsReceiver.counter("writesFailed") private val twhinVideoEmbeddingFutureObserver = FutureObserver( scopedStatsReceiver.scope("twhinVideoEmbedding")) private val twhinVideoEmbeddingOptionObserver = OptionObserver( scopedStatsReceiver.scope("twhinVideoEmbedding")) private val clipEmbeddingCounter = scopedStatsReceiver.counter("clipEmbeddingCounter") private val tweetWatchTimeMetadataRequestCounter = scopedStatsReceiver.counter("tweetWatchTimeMetadataRequests") private val tweetWatchTimeMetadataSuccessCounter = scopedStatsReceiver.counter("tweetWatchTimeMetadataSuccessCounter") private val tweetWatchTimeMetadataFailureCounter = scopedStatsReceiver.counter("tweetWatchTimeMetadataFailureCounter") private val drMerger = new DataRecordMerger private val predictedScoreFeaturesDataRecordAdapter = new DataRecordConverter(SpecificFeatures(PredictedScoreFeatureSet)) lazy private val dataRecordMetadataStoreClient: Option[Client with Transactions] = Try { try { val c = MysqlClientUtils.parseConfigFromYaml(dataRecordMetadataStoreConfigsYml) logger.info(s"pldr mysql config: ${c.host} ${c.port} ${c.user} ${c.database}") } catch { case e: Throwable => logger.error("pldr mysql error: " + e.toString) } MysqlClientUtils.mysqlClientProvider( MysqlClientUtils.parseConfigFromYaml(dataRecordMetadataStoreConfigsYml) ) }.toOption lazy private val versionedMetadataCacheClientOpt: Option[ VersionedMetadataCacheClient[Map[drmd.FeaturesCategory, Option[VersionIdAndFeatures]]] ] = dataRecordMetadataStoreClient.map { mysqlClient => new VersionedMetadataCacheClient[Map[drmd.FeaturesCategory, Option[VersionIdAndFeatures]]]( maximumSize = 1, expireDurationOpt = None, mysqlClient = mysqlClient, transform = CandidateAndCommonFeaturesStreamingUtils.metadataTransformer, statsReceiver = statsReceiver ) } versionedMetadataCacheClientOpt.foreach { _.metadataFetchTimerTask( CandidateAndCommonFeaturesStreamingUtils.metadataFetchKey, metadataFetchTimer = DefaultTimer, metadataFetchInterval = 90.seconds, metadataFetchFailedCounter = metadataFetchFailedCounter ) } override def apply( inputs: PipelineResultSideEffect.Inputs[PipelineQuery, HasMarshalling] ): Stitch[Unit] = Stitch.let(OffloadFuturePool.lowPriorityLocal)(()) { OffloadFuturePools.offloadStitch { // Exclude unscored candidates (e.g. because of scoring QF truncation or cache reads) val selectedCandidates = (inputs.selectedCandidates ++ inputs.remainingCandidates) .filter { candidate => candidate.features.contains(CandidateFeaturesDataRecordFeature) } val droppedCandidates = inputs.droppedCandidates .filter { candidate => candidate.features.contains(CandidateFeaturesDataRecordFeature) } val candidates = (selectedCandidates ++ droppedCandidates) val isSelectedCandidateIds: Set[Long] = selectedCandidates.map(_.candidateIdLong).toSet val candidatesHeavyRankerScoreBasedRank: Map[Long, Int] = candidates .sortBy(-_.features .getOrElse(WeightedModelScoreFeature, None).getOrElse(Double.NegativeInfinity)).map( _.candidateIdLong).zipWithIndex.toMap val isSelectedByHeavyRankerCount = inputs.query.params(IsSelectedByHeavyRankerCountParam) val predictionRequestId = candidates.headOption.flatMap { candidate => candidate.features.getOrElse(PredictionRequestIdFeature, None) } val productSurface = inputs.query.product match { case FollowingProduct => hmt.Product.Following case ForYouProduct => hmt.Product.ForYou case ScoredTweetsProduct => hmt.Product.ScoredTweets case ScoredVideoTweetsProduct => hmt.Product.ScoredVideoTweets case SubscribedProduct => hmt.Product.Subscribed case other => throw new UnsupportedOperationException(s"Unknown product: $other") } val nonMLCommonFeatures = NonMLCommonFeatures( userId = inputs.query.getRequiredUserId, guestId = inputs.query.features.flatMap(_.getOrElse(GuestIdFeature, None)), clientId = inputs.query.features.flatMap(_.getOrElse(ClientIdFeature, None)), countryCode = inputs.query.getCountryCode, predictionRequestId = predictionRequestId, productSurface = productSurface.toString, servedTimestamp = inputs.query.queryTime.inMilliseconds ) val nonMLCommonFeaturesDataRecord = NonMLCommonFeaturesAdapter.adaptToDataRecords(nonMLCommonFeatures).asScala.head val sideEffectStitch = fetchExperimentalFeatures(inputs.query, candidates) .map { candidatesAndFeatures => // Further writes are cheap and do not have callbacks, // therefore it's safe to bypass offload filter. OffloadFilter.withOffloadsDisabled { candidatesAndFeatures.foreach { case (candidate, experimentalFeatures) => val candidateFeaturesPldr = buildCandidateFeaturesPldr( query = inputs.query, candidate = candidate, isSelected = isSelectedCandidateIds.contains(candidate.candidateIdLong), isSelectedByHeavyRanker = candidatesHeavyRankerScoreBasedRank .getOrElse( candidate.candidateIdLong, candidates.size) < isSelectedByHeavyRankerCount, rankByHeavyRanker = candidatesHeavyRankerScoreBasedRank .getOrElse(candidate.candidateIdLong, candidates.size), nonMLCommonFeaturesDataRecord = nonMLCommonFeaturesDataRecord, experimentalFeaturesDataRecords = experimentalFeatures, ) writesRequestCounter.incr() val candidateFeaturesKey = sc.CandidateFeatureKey( tweetId = candidate.candidateIdLong, viewerId = inputs.query.getRequiredUserId, servedId = predictionRequestId.getOrElse(-1L) ) store.put(candidateFeaturesKey -> candidateFeaturesPldr).rescue { case _: Throwable => writesFailedCounter.incr() Future.Unit } } } } Stitch.run(sideEffectStitch) Stitch.Unit } } private def buildCandidateFeaturesPldr( query: PipelineQuery, candidate: CandidateWithDetails, isSelected: Boolean, isSelectedByHeavyRanker: Boolean, rankByHeavyRanker: Int, nonMLCommonFeaturesDataRecord: DataRecord, experimentalFeaturesDataRecords: Seq[DataRecord] ): Option[pldr.PolyDataRecord] = { // Step 1) Set candidate features to all existing candidate features used in ranking val candidateFeaturesDataRecord = candidate.features.get(CandidateFeaturesDataRecordFeature) // Step 2) Remove all large embeddings features from DataRecord AllCandidateLargeEmbeddingsFeatures.foreach { feature => candidateFeaturesDataRecord.tensors.remove(feature.getFeatureId) } // Step 3) Add prediction score val predictedScoreFeaturesDataRecord = predictedScoreFeaturesDataRecordAdapter.toDataRecord(candidate.features) drMerger.merge(candidateFeaturesDataRecord, predictedScoreFeaturesDataRecord) // Step 4) Add non-ML common features drMerger.merge(candidateFeaturesDataRecord, nonMLCommonFeaturesDataRecord) // Step 5) Add non-ML candidate features, including light ranking features val nonMLCandidateFeatures = NonMLCandidateFeatures( tweetId = candidate.candidateIdLong, sourceTweetId = getOriginalTweetId(candidate.candidateIdLong, candidate.features), originalAuthorId = getOriginalAuthorId(candidate.features) ) val nonMLCandidateFeaturesDataRecord = NonMLCandidateFeaturesAdapter.adaptToDataRecords(nonMLCandidateFeatures).asScala.head val lightRankingCandidateFeatures = LightRankingCandidateFeatures( isSelected = isSelected, isSelectedByHeavyRanker = isSelectedByHeavyRanker, rankByHeavyRanker = rankByHeavyRanker, servedType = candidate.features.get(ServedTypeFeature), candidateSourcePosition = candidate.features.get(CandidateSourcePosition).toLong ) val lightRankingCandidateFeaturesDataRecord = LightRankingCandidateFeaturesAdapter .adaptToDataRecords(lightRankingCandidateFeatures).asScala.head drMerger.merge(nonMLCandidateFeaturesDataRecord, lightRankingCandidateFeaturesDataRecord) drMerger.merge(candidateFeaturesDataRecord, nonMLCandidateFeaturesDataRecord) // Step 5) Add experimental features (including twhin) experimentalFeaturesDataRecords.foreach(drMerger.merge(candidateFeaturesDataRecord, _)) CandidateAndCommonFeaturesStreamingUtils.candidateFeaturesToPolyDataRecord( versionedMetadataCacheClientOpt = versionedMetadataCacheClientOpt, candidateFeatures = candidateFeaturesDataRecord, valueFormat = pldr.PolyDataRecord._Fields.LITE_COMPACT_DATA_RECORD ) } private def fetchExperimentalFeatures( query: PipelineQuery, candidates: Seq[CandidateWithDetails], ): Stitch[Map[CandidateWithDetails, Seq[DataRecord]]] = { val stitches = Seq( fetchTwhinVideoEmbeddingDataRecords(query, candidates), fetchTweetTextTokensDataRecord(query, candidates), fetchClipEmbeddings(query, candidates), fetchTweetVideoAggregatedWatchTimeDataRecord(query, candidates), ) Stitch .collect(stitches) .map { candidatesMaps => val candidatesToDataRecords = new java.util.HashMap[CandidateWithDetails, ArrayBuffer[DataRecord]]( candidates.size * 3 / 4) candidatesMaps.foreach { candidatesToDataRecord => candidatesToDataRecord.foreach { case (candidate, dataRecord) => if (dataRecord.isDefined) { candidatesToDataRecords .computeIfAbsent( candidate, _ => new ArrayBuffer[DataRecord](stitches.size) ).append(dataRecord.get) } } } candidatesToDataRecords.asScala.toMap } } private def fetchTwhinVideoEmbeddingDataRecords( query: PipelineQuery, candidates: Seq[CandidateWithDetails], ): Stitch[Map[CandidateWithDetails, Option[DataRecord]]] = { if (!query.params(EnableTwhinVideoFeaturesParam)) { Stitch.value(Map.empty) } else { val originalTweetToCandidates = candidates.groupBy { CandidatesUtil.getOriginalTweetId(_) } Stitch.callFuture { twhinVideoEmbeddingFutureObserver( Future.collect(twhinVideoStore.multiGet(originalTweetToCandidates.keySet)).map { originalTweetIdToEmbeddings => originalTweetIdToEmbeddings.flatMap { case (originalTweetId, embeddingOpt) => val floatTensor = twhinVideoEmbeddingOptionObserver(embeddingOpt) .map { e => ml.FloatTensor(e.embedding) } val dataRecord = TwhinVideoEmbeddingsAdapter.adaptToDataRecords(floatTensor).asScala.headOption originalTweetToCandidates(originalTweetId).map { candidate => candidate -> dataRecord } } } ) } } } private def fetchClipEmbeddings( query: PipelineQuery, candidates: Seq[CandidateWithDetails], ): Stitch[Map[CandidateWithDetails, Option[DataRecord]]] = { Stitch .collect( candidates.map { candidate => if (query.params(EnableClipEmbeddingFeaturesParam) && candidate.features.getOrElse(HasVideoFeature, false) && !query.params( EnableVideoClipEmbeddingFeatureHydrationDeciderParam )) { // If it's not enabled through feature hydrator clipEmbeddingCounter.incr() twitterClipEmbeddingMhClientColumn.fetcher .fetch(candidate.candidateIdLong) .map { result => candidate -> result.v.flatMap { record => ClipEmbeddingFeaturesAdapter .adaptToDataRecords(record) .asScala .headOption } } } else { Stitch.value(candidate -> None) } } ).map(_.toMap) } private def fetchTweetTextTokensDataRecord( query: PipelineQuery, candidates: Seq[CandidateWithDetails], ): Stitch[Map[CandidateWithDetails, Option[DataRecord]]] = { Stitch.value { if (query.params(EnableTweetTextTokensEmbeddingFeatureScribingParam)) { Map.empty } else { candidates.map { candidate => candidate -> candidate.features.getOrElse(TweetTextTokensFeature, None).flatMap { textTokens => TextTokensFeaturesAdapter .adaptToDataRecords(textTokens) .asScala .headOption } }.toMap } } } private def fetchTweetVideoAggregatedWatchTimeDataRecord( query: PipelineQuery, candidates: Seq[CandidateWithDetails], ): Stitch[Map[CandidateWithDetails, Option[DataRecord]]] = { Stitch .collect( candidates.map { candidate => if (query.params( EnableTweetVideoAggregatedWatchTimeFeatureScribingParam) && candidate.features .getOrElse(HasVideoFeature, false)) { tweetWatchTimeMetadataRequestCounter.incr() Stitch.callFuture( tweetWatchTimeMetadataStore .get( (candidate.candidateIdLong, ts.VideoViewEngagementType.ImmersiveVideoWatchTime) ).onSuccess { _ => tweetWatchTimeMetadataSuccessCounter.incr() } .onFailure { _ => tweetWatchTimeMetadataFailureCounter.incr() } .map { opt => candidate -> opt.flatMap(VideoAggregatedWatchTimeFeaturesAdapter .adaptToDataRecords(_).asScala.headOption) } ) } else { Stitch.value(candidate -> None) } } ).map(_.toMap) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/ClientEventsBuilder.scala ================================================ package com.twitter.home_mixer.functional_component.side_effect import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.functional_component.decorator.HomeQueryTypePredicates import com.twitter.home_mixer.functional_component.decorator.builder.HomeTweetTypePredicates import com.twitter.home_mixer.model.HomeFeatures.AccountAgeFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTypeMetricsFeature import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.ClientEvent import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect.EventNamespace import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.suggests.controller_data.Home private[side_effect] sealed trait ClientEventsBuilder { private val FollowingSection = Some("latest") private val ForYouSection = Some("home") private val SubscribedSection = Some("subscribed") protected def section(query: PipelineQuery): Option[String] = { query.product match { case FollowingProduct => FollowingSection case ForYouProduct => ForYouSection case SubscribedProduct => SubscribedSection case other => throw new UnsupportedOperationException(s"Unknown product: $other") } } protected def count( candidates: Seq[CandidateWithDetails], predicate: FeatureMap => Boolean = _ => true, queryFeatures: FeatureMap = FeatureMap.empty ): Option[Long] = Some(candidates.view.count(item => predicate(item.features ++ queryFeatures))) protected def sum( candidates: Seq[CandidateWithDetails], predicate: FeatureMap => Option[Int], queryFeatures: FeatureMap = FeatureMap.empty ): Option[Long] = Some(candidates.view.flatMap(item => predicate(item.features ++ queryFeatures)).sum) } private[side_effect] object ServedEventsBuilder extends ClientEventsBuilder { private val ServedTweetsAction = Some("served_tweets") private val ServedUsersAction = Some("served_users") private val ServedCommunitiesAction = Some("served_communities") private val ServedPromptsAction = Some("served_prompts") private val InjectedComponent = Some("injected") private val PromotedComponent = Some("promoted") private val WhoToFollowComponent = Some("who_to_follow") private val WhoToSubscribeComponent = Some("who_to_subscribe") private val CommunitiesToJoinComponent = Some("communities_to_join") private val RelevancePromptComponent = Some("for_you_survey_feed") private val WithVideoDurationComponent = Some("with_video_duration") private val VideoDurationSumElement = Some("video_duration_sum") private val NumVideosElement = Some("num_videos") def tweetTypePredicate(predicateName: String): FeatureMap => Boolean = { val predicateIdxOpt = Home.ItemTypeIdxMap.get(predicateName) if (predicateIdxOpt.nonEmpty) { val predicateIdx = predicateIdxOpt.get featureMap: FeatureMap => { featureMap .getOrElse(TweetTypeMetricsFeature, None) .map { tweetTypeMetrics => java.util.BitSet .valueOf(tweetTypeMetrics.toArray) .get(predicateIdx) }.getOrElse(false) } } else _ => false } def build( query: PipelineQuery, injectedTweets: Seq[ItemCandidateWithDetails], promotedTweets: Seq[ItemCandidateWithDetails], whoToFollowUsers: Seq[ItemCandidateWithDetails], whoToSubscribeUsers: Seq[ItemCandidateWithDetails], communititesToJoin: Seq[ItemCandidateWithDetails], relevancePrompt: Seq[ItemCandidateWithDetails] ): Seq[ClientEvent] = { val baseEventNamespace = EventNamespace( section = section(query), action = ServedTweetsAction ) val overallServedEvents = Seq( ClientEvent(baseEventNamespace, eventValue = count(injectedTweets ++ promotedTweets)), ClientEvent( baseEventNamespace.copy(component = InjectedComponent), eventValue = count(injectedTweets)), ClientEvent( baseEventNamespace.copy(component = PromotedComponent), eventValue = count(promotedTweets)), ClientEvent( baseEventNamespace.copy(component = WhoToFollowComponent, action = ServedUsersAction), eventValue = count(whoToFollowUsers)), ClientEvent( baseEventNamespace.copy(component = WhoToSubscribeComponent, action = ServedUsersAction), eventValue = count(whoToSubscribeUsers)), ClientEvent( baseEventNamespace .copy(component = CommunitiesToJoinComponent, action = ServedCommunitiesAction), eventValue = count(communititesToJoin)), ClientEvent( baseEventNamespace .copy(component = RelevancePromptComponent, action = ServedPromptsAction), eventValue = count(relevancePrompt)), ) val tweetTypeServedEvents = HomeTweetTypePredicates.PredicateMap.map { case (tweetType, predicate) => ClientEvent( baseEventNamespace.copy(component = InjectedComponent, element = Some(tweetType)), eventValue = count( injectedTweets, tweetTypePredicate(tweetType), query.features.getOrElse(FeatureMap.empty)) ) }.toSeq val servedTypeServedEvents = injectedTweets .map(_.features.get(ServedTypeFeature)) .groupBy(identity).map { case (servedType, group) => ClientEvent( baseEventNamespace.copy(component = Some(servedType.originalName)), eventValue = Some(group.size.toLong)) }.toSeq // Video duration events val numVideosEvent = ClientEvent( baseEventNamespace.copy(component = WithVideoDurationComponent, element = NumVideosElement), eventValue = count(injectedTweets, _.getOrElse(VideoDurationMsFeature, None).nonEmpty) ) val videoDurationSumEvent = ClientEvent( baseEventNamespace .copy(component = WithVideoDurationComponent, element = VideoDurationSumElement), eventValue = sum(injectedTweets, _.getOrElse(VideoDurationMsFeature, None)) ) val videoEvents = Seq(numVideosEvent, videoDurationSumEvent) overallServedEvents ++ tweetTypeServedEvents ++ servedTypeServedEvents ++ videoEvents } } private[side_effect] object EmptyTimelineEventsBuilder extends ClientEventsBuilder { private val EmptyAction = Some("empty") private val AccountAgeLessThan30MinutesComponent = Some("account_age_less_than_30_minutes") private val ServedNonPromotedTweetElement = Some("served_non_promoted_tweet") def build( query: PipelineQuery, injectedTweets: Seq[ItemCandidateWithDetails] ): Seq[ClientEvent] = { val baseEventNamespace = EventNamespace( section = section(query), action = EmptyAction ) // Empty timeline events val accountAgeLessThan30Minutes = query.features .flatMap(_.getOrElse(AccountAgeFeature, None)) .exists(_.untilNow < 30.minutes) val isEmptyTimeline = count(injectedTweets).contains(0L) val predicates = Seq( None -> isEmptyTimeline, AccountAgeLessThan30MinutesComponent -> (isEmptyTimeline && accountAgeLessThan30Minutes) ) for { (component, predicate) <- predicates if predicate } yield ClientEvent( baseEventNamespace.copy(component = component, element = ServedNonPromotedTweetElement)) } } private[side_effect] object QueryEventsBuilder extends ClientEventsBuilder { private val ServedSizePredicateMap: Map[String, Int => Boolean] = Map( ("size_is_empty", _ <= 0), ("size_at_most_5", _ <= 5), ("size_at_most_10", _ <= 10), ("size_at_most_35", _ <= 35) ) def build( query: PipelineQuery, injectedTweets: Seq[ItemCandidateWithDetails] ): Seq[ClientEvent] = { val baseEventNamespace = EventNamespace( section = section(query) ) val queryFeatureMap = query.features.getOrElse(FeatureMap.empty) val servedSizeQueryEvents = for { (queryPredicateName, queryPredicate) <- HomeQueryTypePredicates.PredicateMap if queryPredicate(queryFeatureMap) (servedSizePredicateName, servedSizePredicate) <- ServedSizePredicateMap if servedSizePredicate(injectedTweets.size) } yield ClientEvent( baseEventNamespace .copy(component = Some(servedSizePredicateName), action = Some(queryPredicateName))) servedSizeQueryEvents.toSeq } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/CommonFeaturesPldrConverter.scala ================================================ package com.twitter.home_mixer.functional_component.side_effect import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mysql.Client import com.twitter.finagle.mysql.Transactions import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.util.DefaultTimer import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features.NonMLCommonFeatures import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.non_ml_features.NonMLCommonFeaturesAdapter import com.twitter.home_mixer.functional_component.scorer.CommonFeaturesDataRecordFeature import com.twitter.home_mixer.model.HomeFeatures.ClientIdFeature import com.twitter.home_mixer.model.HomeFeatures.GuestIdFeature import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.ScoredTweetsProduct import com.twitter.home_mixer.model.request.ScoredVideoTweetsProduct import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.home_mixer.param.HomeGlobalParams import com.twitter.home_mixer.param.HomeMixerFlagName.DataRecordMetadataStoreConfigsYmlFlag import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.inject.annotations.Flag import com.twitter.ml.api.DataRecordMerger import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.CandidateAndCommonFeaturesStreamingUtils import com.twitter.timelines.ml.pldr.client.MysqlClientUtils import com.twitter.timelines.ml.pldr.client.VersionedMetadataCacheClient import com.twitter.timelines.ml.pldr.conversion.VersionIdAndFeatures import com.twitter.timelines.prediction.features.large_embeddings.LargeEmbeddingsFeatures.AllCommonLargeEmbeddingsFeatures import com.twitter.timelines.suggests.common.data_record_metadata.{thriftscala => drmd} import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} import com.twitter.timelines.util.stats.OptionObserver import com.twitter.util.Try import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ @Singleton class CommonFeaturesPldrConverter @Inject() ( @Flag(DataRecordMetadataStoreConfigsYmlFlag) dataRecordMetadataStoreConfigsYml: String, statsReceiver: StatsReceiver) { private val drMerger = new DataRecordMerger private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val metadataFetchFailedCounter = scopedStatsReceiver.counter("metadataFetchFailed") private val commonFeaturesPLDROptionObserver = OptionObserver(scopedStatsReceiver.scope("commonFeaturesPLDR")) private lazy val dataRecordMetadataStoreClient: Option[Client with Transactions] = Try { MysqlClientUtils.mysqlClientProvider( MysqlClientUtils.parseConfigFromYaml(dataRecordMetadataStoreConfigsYml) ) }.toOption private lazy val versionedMetadataCacheClientOpt: Option[ VersionedMetadataCacheClient[Map[drmd.FeaturesCategory, Option[VersionIdAndFeatures]]] ] = dataRecordMetadataStoreClient.map { mysqlClient => new VersionedMetadataCacheClient[Map[drmd.FeaturesCategory, Option[VersionIdAndFeatures]]]( maximumSize = 1, expireDurationOpt = None, mysqlClient = mysqlClient, transform = CandidateAndCommonFeaturesStreamingUtils.metadataTransformer, statsReceiver = statsReceiver ) } versionedMetadataCacheClientOpt.foreach { _.metadataFetchTimerTask( CandidateAndCommonFeaturesStreamingUtils.metadataFetchKey, metadataFetchTimer = DefaultTimer, metadataFetchInterval = 90.seconds, metadataFetchFailedCounter = metadataFetchFailedCounter ) } /** * Get the common features data record converted to PLDR format with prediction request ID * * @param query * @param selectedCandidates * @return prediction request ID and the common features PLDR for the given request */ def getCommonFeaturesPldr( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails] ): Option[(Long, pldr.PolyDataRecord)] = { // Exclude unscored candidates (e.g. because of scoring QF truncation or cache reads) val candidatesHead = selectedCandidates .find { candidate => candidate.features.contains(CommonFeaturesDataRecordFeature) } if (candidatesHead.nonEmpty) { val candidateFeaturesHead = candidatesHead.get.features val predictionRequestId = candidateFeaturesHead.getOrElse(PredictionRequestIdFeature, None) val productSurface = query.product match { case FollowingProduct => hmt.Product.Following case ForYouProduct => hmt.Product.ForYou case ScoredTweetsProduct => hmt.Product.ScoredTweets case ScoredVideoTweetsProduct => hmt.Product.ScoredVideoTweets case SubscribedProduct => hmt.Product.Subscribed case other => throw new UnsupportedOperationException(s"Unknown product: $other") } val nonMLCommonFeatures = NonMLCommonFeatures( userId = query.getRequiredUserId, guestId = candidateFeaturesHead.getOrElse(GuestIdFeature, None), clientId = candidateFeaturesHead.getOrElse(ClientIdFeature, None), countryCode = query.getCountryCode, predictionRequestId = predictionRequestId, productSurface = productSurface.toString, servedTimestamp = query.queryTime.inMilliseconds ) val nonMLCommonFeaturesDataRecord = NonMLCommonFeaturesAdapter.adaptToDataRecords(nonMLCommonFeatures).asScala.head val commonFeaturesDataRecord = candidateFeaturesHead.get(CommonFeaturesDataRecordFeature) //Remove large embeddings from dataRecord AllCommonLargeEmbeddingsFeatures.foreach { feature => commonFeaturesDataRecord.tensors.remove(feature.getFeatureId) } drMerger.merge(commonFeaturesDataRecord, nonMLCommonFeaturesDataRecord) val commonFeaturesPLDROpt = CandidateAndCommonFeaturesStreamingUtils .commonFeaturesToPolyDataRecord( versionedMetadataCacheClientOpt = versionedMetadataCacheClientOpt, commonFeatures = commonFeaturesDataRecord, valueFormat = pldr.PolyDataRecord._Fields.LITE_COMPACT_DATA_RECORD, createNew = query.params( HomeGlobalParams.EnableCommonFeaturesDataRecordCopyDuringPldrConversionParam) ) commonFeaturesPLDROptionObserver(commonFeaturesPLDROpt).flatMap { pldr => predictionRequestId.map(_ -> pldr) } } else None } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeClientEventSideEffect.scala ================================================ package com.twitter.home_mixer.functional_component.side_effect import com.twitter.clientapp.thriftscala.LogEvent import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.side_effect.ScribeClientEventSideEffect import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.pipeline.PipelineQuery /** * Side effect that logs served tweet metrics to Scribe as client events. */ case class HomeScribeClientEventSideEffect( enableScribeClientEvents: Boolean, override val logPipelinePublisher: EventPublisher[LogEvent], injectedTweetsCandidatePipelineIdentifiers: Seq[CandidatePipelineIdentifier], adsCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None, whoToFollowCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None, whoToSubscribeCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None, forYouCommunitiesToJoinCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None, forYouRelevancePromptCandidatePipelineIdentifier: Option[CandidatePipelineIdentifier] = None) extends ScribeClientEventSideEffect[PipelineQuery, Timeline] with PipelineResultSideEffect.Conditionally[ PipelineQuery, Timeline ] { override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeClientEvent") override val page = "home" override def onlyIf( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: Timeline ): Boolean = enableScribeClientEvents override def buildClientEvents( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: Timeline ): Seq[ScribeClientEventSideEffect.ClientEvent] = { val itemCandidates = CandidatesUtil.getItemCandidates(selectedCandidates) val sources = itemCandidates.groupBy(_.source) val injectedTweets = injectedTweetsCandidatePipelineIdentifiers.flatMap(sources.getOrElse(_, Seq.empty)) val promotedTweets = adsCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten // WhoToFollow and WhoToSubscribe modules are not required for all home-mixer products, e.g. list tweets timeline. val whoToFollowUsers = whoToFollowCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten val whoToSubscribeUsers = whoToSubscribeCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten val communititesToJoin = forYouCommunitiesToJoinCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten val relevancePrompt = forYouRelevancePromptCandidatePipelineIdentifier.flatMap(sources.get).toSeq.flatten val servedEvents = ServedEventsBuilder .build( query, injectedTweets, promotedTweets, whoToFollowUsers, whoToSubscribeUsers, communititesToJoin, relevancePrompt) val emptyTimelineEvents = EmptyTimelineEventsBuilder.build(query, injectedTweets) val queryEvents = QueryEventsBuilder.build(query, injectedTweets) (servedEvents ++ emptyTimelineEvents ++ queryEvents).filter(_.eventValue.forall(_ > 0)) } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/HomeScribeServedCandidatesSideEffect.scala ================================================ package com.twitter.home_mixer.functional_component.side_effect import com.twitter.finagle.tracing.Trace import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetDetailsMarshaller import com.twitter.home_mixer.marshaller.timeline_logging.TweetDetailsMarshaller import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowDetailsMarshaller import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceSignalFeature import com.twitter.home_mixer.model.HomeFeatures.UserActionsByteArrayFeature import com.twitter.home_mixer.model.HomeFeatures.UserActionsContainsExplicitSignalsFeature import com.twitter.home_mixer.model.HomeFeatures.UserActionsSizeFeature import com.twitter.home_mixer.model.PhoenixPredictedScoreFeature import com.twitter.home_mixer.model.PredictedScoreFeature import com.twitter.home_mixer.model.request.DeviceContext.RequestContext import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.model.request.HasSeenTweetIds import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.home_mixer.param.HomeGlobalParams.EnableScribeServedCandidatesParam import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixInferenceClusterParam import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCandidatesFlag import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator import com.twitter.product_mixer.component_library.side_effect.ScribeLogEventSideEffect import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.timeline_logging.thriftscala.Prediction import com.twitter.timelines.timeline_logging.{thriftscala => thrift} import com.twitter.util.Time import java.nio.ByteBuffer import javax.inject.Inject import javax.inject.Singleton /** * Side effect that logs home timeline served candidates to Scribe. */ @Singleton class HomeScribeServedCandidatesSideEffect @Inject() ( @Flag(ScribeServedCandidatesFlag) enableScribeServedCandidates: Boolean, scribeEventPublisher: EventPublisher[thrift.ServedEntry]) extends ScribeLogEventSideEffect[ thrift.ServedEntry, PipelineQuery with HasSeenTweetIds with HasDeviceContext, Timeline ] with PipelineResultSideEffect.Conditionally[ PipelineQuery with HasSeenTweetIds with HasDeviceContext, Timeline ] { override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedCandidates") override def onlyIf( query: PipelineQuery with HasSeenTweetIds with HasDeviceContext, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: Timeline ): Boolean = enableScribeServedCandidates && query.params(EnableScribeServedCandidatesParam) override def buildLogEvents( query: PipelineQuery with HasSeenTweetIds with HasDeviceContext, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: Timeline ): Seq[thrift.ServedEntry] = { val timelineType = query.product match { case FollowingProduct => thrift.TimelineType.HomeLatest case ForYouProduct => thrift.TimelineType.Home case SubscribedProduct => thrift.TimelineType.HomeSubscribed case other => throw new UnsupportedOperationException(s"Unknown product: $other") } val requestProvenance = query.deviceContext.map { deviceContext => deviceContext.requestContextValue match { case RequestContext.Foreground => thrift.RequestProvenance.Foreground case RequestContext.Launch => thrift.RequestProvenance.Launch case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr case _ => thrift.RequestProvenance.Other } } val queryType = query.features.map { featureMap => if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial else thrift.QueryType.Other } val phoenixCluster = s"phoenix.${query.params(PhoenixInferenceClusterParam).toString.toLowerCase}." val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] = selectedCandidates.flatMap { case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] => Seq((item.candidateIdLong, item)) case module: ModuleCandidateWithDetails if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) => module.candidates.map(item => (item.candidateIdLong, item)) case _ => Seq.empty }.toMap val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] = selectedCandidates.flatMap { case module: ModuleCandidateWithDetails if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) => module.candidates.map { item => (item.candidateIdLong, item) } case _ => Seq.empty }.toMap val userActions = selectedCandidates.collectFirst { case candidate if candidate.features.getOrElse(UserActionsSizeFeature, None).isDefined => ( candidate.features.getOrElse(UserActionsSizeFeature, None), candidate.features.getOrElse(UserActionsContainsExplicitSignalsFeature, false) ) } val userActionsByteArrayOpt = query.features.flatMap(_.getOrElse(UserActionsByteArrayFeature, None)) val userActionsBuffer = userActionsByteArrayOpt.map(ua => ByteBuffer.wrap(ua)) val requestInfo = thrift.RequestInfo( requestTimeMs = query.queryTime.inMilliseconds, traceId = Trace.id.traceId.toLong, userId = query.getOptionalUserId, clientAppId = query.clientContext.appId, hasDarkRequest = query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)), parentId = Some(Trace.id.parentId.toLong), spanId = Some(Trace.id.spanId.toLong), timelineType = Some(timelineType), ipAddress = query.clientContext.ipAddress, userAgent = query.clientContext.userAgent, queryType = queryType, requestProvenance = requestProvenance, languageCode = query.clientContext.languageCode, countryCode = query.clientContext.countryCode, requestEndTimeMs = Some(Time.now.inMilliseconds), servedRequestId = query.features.flatMap(_.getOrElse(ServedIdFeature, None)), requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)), userActionsSize = userActions.flatMap(_._1), userActionsContainsExplicitSignals = userActions.map(_._2), userActions = userActionsBuffer ) response.instructions .collect { case AddEntriesTimelineInstruction(entries) => entries.zipWithIndex.collect { case (entry: TweetItem, index) if entry.promotedMetadata.isDefined => val promotedTweetDetails = PromotedTweetDetailsMarshaller(entry, index) Seq( thrift.EntryInfo( id = entry.id, position = index.shortValue(), entryId = entry.entryIdentifier, entryType = thrift.EntryType.PromotedTweet, sortIndex = entry.sortIndex, verticalSize = Some(1), displayType = Some(entry.displayType.toString), details = Some(thrift.ItemDetails.PromotedTweetDetails(promotedTweetDetails)) )) case (entry: TweetItem, index) => val candidate = tweetIdToItemCandidateMap(entry.id) val tweetDetails = TweetDetailsMarshaller(entry, candidate) Seq( thrift.EntryInfo( id = candidate.candidateIdLong, position = index.shortValue(), entryId = entry.entryIdentifier, entryType = thrift.EntryType.Tweet, sortIndex = entry.sortIndex, verticalSize = Some(1), score = candidate.features.getOrElse(ScoreFeature, None), displayType = Some(entry.displayType.toString), details = Some(thrift.ItemDetails.TweetDetails(tweetDetails)), predictionScores = Some(extractPredictionScores(candidate, phoenixCluster)), sourceSignal = candidate.features.getOrElse(SourceSignalFeature, None).map { signal => thrift.SourceSignal( id = Some(signal.id), signalType = signal.signalType, signalEntity = signal.signalEntity, authorId = signal.authorId, ) } )) case (module: TimelineModule, _) if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString => module.items.zipWithIndex.collect { case (ModuleItem(entry: UserItem, _, _, _), index) => val candidate = userIdToItemCandidateMap(entry.id) val whoToFollowDetails = WhoToFollowDetailsMarshaller(entry, candidate) thrift.EntryInfo( id = entry.id, position = index.shortValue(), entryId = module.entryIdentifier, entryType = thrift.EntryType.WhoToFollowModule, sortIndex = module.sortIndex, score = candidate.features.getOrElse(ScoreFeature, None), displayType = Some(entry.displayType.toString), details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToFollowDetails)) ) } case (module: TimelineModule, _) if module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString => module.items.zipWithIndex.collect { case (ModuleItem(entry: UserItem, _, _, _), index) => val candidate = userIdToItemCandidateMap(entry.id) val whoToSubscribeDetails = WhoToFollowDetailsMarshaller(entry, candidate) thrift.EntryInfo( id = entry.id, position = index.shortValue(), entryId = module.entryIdentifier, entryType = thrift.EntryType.WhoToSubscribeModule, sortIndex = module.sortIndex, score = candidate.features.getOrElse(ScoreFeature, None), displayType = Some(entry.displayType.toString), details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToSubscribeDetails)) ) } case (module: TimelineModule, _) if module.sortIndex.isDefined && module.items.headOption.exists( _.item.isInstanceOf[TweetItem]) => module.items.zipWithIndex.collect { case (ModuleItem(entry: TweetItem, _, _, _), index) => val candidate = tweetIdToItemCandidateMap(entry.id) thrift.EntryInfo( id = entry.id, position = index.shortValue(), entryId = module.entryIdentifier, entryType = thrift.EntryType.ConversationModule, sortIndex = module.sortIndex, score = candidate.features.getOrElse(ScoreFeature, None), displayType = Some(entry.displayType.toString), predictionScores = Some(extractPredictionScores(candidate, phoenixCluster)), sourceSignal = candidate.features.getOrElse(SourceSignalFeature, None).map { signal => thrift.SourceSignal( id = Some(signal.id), signalType = signal.signalType ) } ) } case _ => Seq.empty }.flatten // Other instructions case _ => Seq.empty[thrift.EntryInfo] }.flatten.map { entryInfo => thrift.ServedEntry( entry = Some(entryInfo), request = requestInfo ) } } override val logPipelinePublisher: EventPublisher[thrift.ServedEntry] = scribeEventPublisher override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert() ) private def extractPredictionScores( candidate: ItemCandidateWithDetails, phoenixCluster: String ): Set[Prediction] = { val naviPredictionScores = PredictedScoreFeature.PredictedScoreFeatureSet.map { feature => Prediction(Some(feature.featureName), candidate.features.getOrElse(feature, None)) } val phoenixPredictionScores = PhoenixPredictedScoreFeature.PhoenixPredictedScoreFeatureSet.map { feature => Prediction( Some(phoenixCluster + feature.featureName), candidate.features.getOrElse(feature, None) ) } (naviPredictionScores ++ phoenixPredictionScores).filter(_.score.isDefined) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsEventBusSideEffect.scala ================================================ package com.twitter.home_mixer.functional_component.side_effect import com.twitter.eventbus.client.EventBusPublisher import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.home_mixer.model.request.HasSeenTweetIds import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.HasMarshalling import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.impressionstore.thriftscala.Impression import com.twitter.timelines.impressionstore.thriftscala.ImpressionList import com.twitter.timelines.impressionstore.thriftscala.PublishedImpressionList import com.twitter.timelines.impressionstore.thriftscala.SurfaceArea import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton object PublishClientSentImpressionsEventBusSideEffect { val HomeSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeTimeline)) val HomeLatestSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeLatestTimeline)) val HomeSubscribedSurfaceArea: Option[Set[SurfaceArea]] = Some(Set(SurfaceArea.HomeSubscribed)) } /** * Side effect that publishes seen tweet IDs sent from clients. The seen tweet IDs are sent to a * heron topology which writes to a memcache dataset. */ @Singleton class PublishClientSentImpressionsEventBusSideEffect @Inject() ( eventBusPublisher: EventBusPublisher[PublishedImpressionList], statsReceiver: StatsReceiver) extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling] with PipelineResultSideEffect.Conditionally[ PipelineQuery with HasSeenTweetIds, HasMarshalling ] { import PublishClientSentImpressionsEventBusSideEffect._ override val identifier: SideEffectIdentifier = SideEffectIdentifier("PublishClientSentImpressionsEventBus") private val seenIdsStatsReceiver = statsReceiver.scope(identifier.toString).scope("SeenIds") override def onlyIf( query: PipelineQuery with HasSeenTweetIds, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: HasMarshalling ): Boolean = query.seenTweetIds.exists(_.nonEmpty) def buildEvents( query: PipelineQuery with HasSeenTweetIds, currentTime: Long ): Option[Seq[Impression]] = { val surfaceArea = query.product match { case ForYouProduct => HomeSurfaceArea case FollowingProduct => HomeLatestSurfaceArea case SubscribedProduct => HomeSubscribedSurfaceArea case _ => None } val device = query.clientContext.appId.getOrElse(0L).toString query.seenTweetIds.map { seenTweetIds => val getNewer = query.features.map(_.getOrElse(GetNewerFeature, false)).getOrElse(false) val getOlder = query.features.map(_.getOrElse(GetOlderFeature, false)).getOrElse(false) val requestType = if (getNewer) "newer" else if (getOlder) "older" else "none" seenIdsStatsReceiver .scope(query.product.identifier.name).scope(requestType).stat(device) .add(seenTweetIds.distinct.size) seenTweetIds.map { tweetId => Impression( tweetId = tweetId, impressionTime = Some(currentTime), surfaceAreas = surfaceArea ) } } } final override def apply( inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling] ): Stitch[Unit] = { val currentTime = Time.now.inMilliseconds val impressions = buildEvents(inputs.query, currentTime) Stitch.callFuture( eventBusPublisher.publish( PublishedImpressionList( inputs.query.getRequiredUserId, ImpressionList(impressions), currentTime ) ) ) } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.4) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishClientSentImpressionsManhattanSideEffect.scala ================================================ package com.twitter.home_mixer.functional_component.side_effect import com.twitter.home_mixer.model.HomeFeatures.TweetImpressionsFeature import com.twitter.home_mixer.model.request.HasSeenTweetIds import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.HasMarshalling import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.impression.{thriftscala => t} import com.twitter.timelines.impressionstore.store.ManhattanTweetImpressionStoreClient import javax.inject.Inject import javax.inject.Singleton /** * Side effect that updates the timelines tweet impression * store (Manhattan) with seen tweet IDs sent from clients */ @Singleton class PublishClientSentImpressionsManhattanSideEffect @Inject() ( manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient) extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling] with PipelineResultSideEffect.Conditionally[ PipelineQuery with HasSeenTweetIds, HasMarshalling ] { override val identifier: SideEffectIdentifier = SideEffectIdentifier("PublishClientSentImpressionsManhattan") override def onlyIf( query: PipelineQuery with HasSeenTweetIds, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: HasMarshalling ): Boolean = query.seenTweetIds.exists(_.nonEmpty) def buildEvents(query: PipelineQuery): Option[(Long, t.TweetImpressionsEntries)] = { query.features.flatMap { featureMap => val impressions = featureMap.getOrElse(TweetImpressionsFeature, Seq.empty) if (impressions.nonEmpty) Some((query.getRequiredUserId, t.TweetImpressionsEntries(impressions))) else None } } final override def apply( inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling] ): Stitch[Unit] = { val events = buildEvents(inputs.query) Stitch .traverse(events) { case (key, value) => manhattanTweetImpressionStoreClient.write(key, value) } .unit } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.4) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/PublishImpressionBloomFilterSideEffect.scala ================================================ package com.twitter.home_mixer.functional_component.side_effect import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature import com.twitter.home_mixer.model.request.HasSeenTweetIds import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.HasMarshalling import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} import javax.inject.Inject import javax.inject.Singleton @Singleton class PublishImpressionBloomFilterSideEffect @Inject() ( bloomFilterClient: ManhattanStoreClient[ blm.ImpressionBloomFilterKey, blm.ImpressionBloomFilterSeq ]) extends PipelineResultSideEffect[PipelineQuery with HasSeenTweetIds, HasMarshalling] with PipelineResultSideEffect.Conditionally[ PipelineQuery with HasSeenTweetIds, HasMarshalling ] { override val identifier: SideEffectIdentifier = SideEffectIdentifier("PublishImpressionBloomFilter") private val SurfaceArea = blm.SurfaceArea.HomeTimeline override def onlyIf( query: PipelineQuery with HasSeenTweetIds, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: HasMarshalling ): Boolean = query.params.getBoolean(EnableImpressionBloomFilter) && query.seenTweetIds.exists(_.nonEmpty) def buildEvents(query: PipelineQuery): Option[blm.ImpressionBloomFilterSeq] = { query.features.flatMap { featureMap => val impressionBloomFilterSeq = featureMap.get(ImpressionBloomFilterFeature) if (impressionBloomFilterSeq.entries.nonEmpty) Some(impressionBloomFilterSeq) else None } } override def apply( inputs: PipelineResultSideEffect.Inputs[PipelineQuery with HasSeenTweetIds, HasMarshalling] ): Stitch[Unit] = { buildEvents(inputs.query) .map { updatedBloomFilterSeq => bloomFilterClient.write( blm.ImpressionBloomFilterKey(inputs.query.getRequiredUserId, SurfaceArea), updatedBloomFilterSeq) }.getOrElse(Stitch.Unit) } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/TruncateTimelinesPersistenceStoreSideEffect.scala ================================================ package com.twitter.home_mixer.functional_component.side_effect import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.param.HomeGlobalParams.TimelinesPersistenceStoreMaxEntriesPerClient import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelinemixer.clients.persistence.TimelineResponseBatchesClient import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 import com.twitter.timelineservice.model.TimelineQuery import com.twitter.timelineservice.model.core.TimelineKind import javax.inject.Inject import javax.inject.Singleton /** * Side effect that truncates entries in the Timelines Persistence store * based on the number of entries per client. */ @Singleton class TruncateTimelinesPersistenceStoreSideEffect @Inject() ( timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3]) extends PipelineResultSideEffect[PipelineQuery, Timeline] { override val identifier: SideEffectIdentifier = SideEffectIdentifier("TruncateTimelinesPersistenceStore") def getResponsesToDelete(query: PipelineQuery): Seq[TimelineResponseV3] = { val responses = query.features.map(_.getOrElse(PersistenceEntriesFeature, Seq.empty)).toSeq.flatten val responsesByClient = responses.groupBy(_.clientPlatform).values.toSeq val maxEntriesPerClient = query.params(TimelinesPersistenceStoreMaxEntriesPerClient) responsesByClient.flatMap { _.sortBy(_.servedTime.inMilliseconds) .foldRight((Seq.empty[TimelineResponseV3], maxEntriesPerClient)) { case (response, (responsesToDelete, remainingCap)) => if (remainingCap > 0) (responsesToDelete, remainingCap - response.entries.size) else (response +: responsesToDelete, remainingCap) } match { case (responsesToDelete, _) => responsesToDelete } } } final override def apply( inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] ): Stitch[Unit] = { val timelineKind = inputs.query.product match { case FollowingProduct => TimelineKind.homeLatest case ForYouProduct => TimelineKind.home case other => throw new UnsupportedOperationException(s"Unknown product: $other") } val timelineQuery = TimelineQuery(id = inputs.query.getRequiredUserId, kind = timelineKind) val responsesToDelete = getResponsesToDelete(inputs.query) if (responsesToDelete.nonEmpty) Stitch.callFuture(timelineResponseBatchesClient.delete(timelineQuery, responsesToDelete)) else Stitch.Unit } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateLastNonPollingTimeSideEffect.scala ================================================ package com.twitter.home_mixer.functional_component.side_effect import com.twitter.home_mixer.model.HomeFeatures.FollowingLastNonPollingTimeFeature import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature import com.twitter.home_mixer.model.HomeFeatures.PollingFeature import com.twitter.home_mixer.model.request.DeviceContext import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.side_effect.UserSessionStoreUpdateSideEffect import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.marshalling.HasMarshalling import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelineservice.model.util.FinagleRequestContext import com.twitter.user_session_store.ReadWriteUserSessionStore import com.twitter.user_session_store.WriteRequest import com.twitter.user_session_store.thriftscala.NonPollingTimestamps import com.twitter.user_session_store.thriftscala.UserSessionField import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton /** * Side effect that updates the User Session Store (Manhattan) with the timestamps of non polling requests. */ @Singleton class UpdateLastNonPollingTimeSideEffect[ Query <: PipelineQuery with HasDeviceContext, ResponseType <: HasMarshalling] @Inject() ( override val userSessionStore: ReadWriteUserSessionStore) extends UserSessionStoreUpdateSideEffect[ WriteRequest, Query, ResponseType ] { private val MaxNonPollingTimes = 10 override val identifier: SideEffectIdentifier = SideEffectIdentifier("UpdateLastNonPollingTime") /** * When the request is non polling and is not a background fetch request, update * the list of non polling timestamps with the timestamp of the current request */ override def buildWriteRequest(query: Query): Option[WriteRequest] = { val isBackgroundFetch = query.deviceContext .exists(_.requestContextValue.contains(DeviceContext.RequestContext.BackgroundFetch)) if (!query.features.exists(_.getOrElse(PollingFeature, false)) && !isBackgroundFetch) { val fields = Seq(UserSessionField.NonPollingTimestamps(makeLastNonPollingTimestamps(query))) Some(WriteRequest(query.getRequiredUserId, fields)) } else None } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.96) ) private def makeLastNonPollingTimestamps(query: Query): NonPollingTimestamps = { val priorNonPollingTimestamps = query.features.map(_.getOrElse(NonPollingTimesFeature, Seq.empty)).toSeq.flatten val lastNonPollingTimeMs = FinagleRequestContext.default.requestStartTime.get.getOrElse(Time.now).inMillis val followingLastNonPollingTime = query.features .flatMap(features => features.getOrElse(FollowingLastNonPollingTimeFeature, None)) .map(_.inMillis) NonPollingTimestamps( nonPollingTimestampsMs = (lastNonPollingTimeMs +: priorNonPollingTimestamps).take(MaxNonPollingTimes), mostRecentHomeLatestNonPollingTimestampMs = if (query.product == FollowingProduct) Some(lastNonPollingTimeMs) else followingLastNonPollingTime ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect/UpdateTimelinesPersistenceStoreSideEffect.scala ================================================ package com.twitter.home_mixer.functional_component.side_effect import com.twitter.home_mixer.functional_component.decorator.EntryPointPivotModuleCandidateDecorator import com.twitter.home_mixer.functional_component.decorator.ForYouTweetCandidateDecorator import com.twitter.home_mixer.functional_component.decorator.KeywordTrendsModuleCandidateDecorator import com.twitter.home_mixer.functional_component.decorator.StoriesModuleCandidateDecorator import com.twitter.home_mixer.functional_component.decorator.TuneFeedModuleCandidateDecorator import com.twitter.home_mixer.functional_component.decorator.VideoCarouselModuleCandidateDecorator import com.twitter.home_mixer.functional_component.decorator.urt.builder.BookmarksModuleCandidateDecorator import com.twitter.home_mixer.functional_component.decorator.urt.builder.PinnedTweetsModuleCandidateDecorator import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.model.PredictedDwellScoreFeature import com.twitter.home_mixer.model.PredictedFavoriteScoreFeature import com.twitter.home_mixer.model.PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature import com.twitter.home_mixer.model.PredictedGoodClickConvoDescUamGt2ScoreFeature import com.twitter.home_mixer.model.PredictedGoodProfileClickScoreFeature import com.twitter.home_mixer.model.PredictedNegativeFeedbackV2ScoreFeature import com.twitter.home_mixer.model.PredictedReplyEngagedByAuthorScoreFeature import com.twitter.home_mixer.model.PredictedReplyScoreFeature import com.twitter.home_mixer.model.PredictedRetweetScoreFeature import com.twitter.home_mixer.model.PredictedShareScoreFeature import com.twitter.home_mixer.model.PredictedVideoQualityViewScoreFeature import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.param.HomeGlobalParams.EnablePersistenceDebug import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.pipeline.candidate.communities_to_join.CommunitiesToJoinCandidateDecorator import com.twitter.product_mixer.component_library.pipeline.candidate.job.RecommendedJobsCandidateDecorator import com.twitter.product_mixer.component_library.pipeline.candidate.recruiting_organization.RecommendedRecruitingOrganizationsCandidateDecorator import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction import com.twitter.product_mixer.core.model.marshalling.response.urt.ReplaceEntryTimelineInstruction import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowCoverInstruction import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule import com.twitter.product_mixer.core.model.marshalling.response.urt.item.prompt.PromptItem import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem import com.twitter.product_mixer.core.model.marshalling.response.urt.promoted.PromotedMetadata import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelinemixer.clients.persistence.EntryWithItemIds import com.twitter.timelinemixer.clients.persistence.ItemIds import com.twitter.timelinemixer.clients.persistence.TimelineResponseBatchesClient import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 import com.twitter.timelines.persistence.{thriftscala => persistence} import com.twitter.timelineservice.model.PredictedScores import com.twitter.timelineservice.model.TimelineQuery import com.twitter.timelineservice.model.TimelineQueryOptions import com.twitter.timelineservice.model.TweetScoreV1 import com.twitter.timelineservice.model.core.TimelineKind import com.twitter.timelineservice.model.rich.EntityIdType import com.twitter.util.Time import com.twitter.{timelineservice => tls} import javax.inject.Inject import javax.inject.Singleton /** * Side effect that updates the Timelines Persistence Store (Manhattan) with the entries being returned. */ @Singleton class UpdateTimelinesPersistenceStoreSideEffect @Inject() ( timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3]) extends PipelineResultSideEffect[PipelineQuery, Timeline] { override val identifier: SideEffectIdentifier = SideEffectIdentifier("UpdateTimelinesPersistenceStore") final override def apply( inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] ): Stitch[Unit] = { if (inputs.response.instructions.nonEmpty) { val timelineKind = inputs.query.product match { case FollowingProduct => TimelineKind.homeLatest case ForYouProduct => TimelineKind.home case other => throw new UnsupportedOperationException(s"Unknown product: $other") } val debugEnabled = inputs.query.params(EnablePersistenceDebug) val timelineQuery = TimelineQuery( id = inputs.query.getRequiredUserId, kind = timelineKind, options = TimelineQueryOptions( contextualUserId = inputs.query.getOptionalUserId, deviceContext = tls.DeviceContext.empty.copy( userAgent = inputs.query.clientContext.userAgent, clientAppId = inputs.query.clientContext.appId) ) ) val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] = inputs.selectedCandidates.flatMap { case item: ItemCandidateWithDetails if item.candidate.id.isInstanceOf[Long] => Seq((item.candidateIdLong, item)) case module: ModuleCandidateWithDetails if module.candidates.headOption.exists(_.candidate.id.isInstanceOf[Long]) => module.candidates.map(item => (item.candidateIdLong, item)) case _ => Seq.empty }.toMap val entries = inputs.response.instructions.collect { case AddEntriesTimelineInstruction(entries) => entries.collect { // includes tweets, tweet previews, and promoted tweets case entry: TweetItem if entry.sortIndex.isDefined => val tweetEntry = buildTweetEntryWithItemIds( tweetIdToItemCandidateMap(entry.id), entry.sortIndex.get, debugEnabled, entry.promotedMetadata ) Seq(tweetEntry) case entry: PromptItem => Seq( EntryWithItemIds( // Temporarily use annotation here as entity type until we come up with a proper one entityIdType = EntityIdType.Annotation, sortIndex = entry.sortIndex.getOrElse(0L), size = 1, itemIds = None )) case module: TimelineModule if module.sortIndex.isDefined && module.items.nonEmpty => if (module.entryNamespace == ForYouTweetCandidateDecorator.ConvoModuleEntryNamespace) { module.items.map { item => buildTweetEntryWithItemIds( tweetIdToItemCandidateMap(item.item.id.asInstanceOf[Long]), module.sortIndex.get, debugEnabled ) } } else if (module.entryNamespace == StoriesModuleCandidateDecorator.TrendsEntryNamespace || module.entryNamespace == KeywordTrendsModuleCandidateDecorator.KeywordTrendsEntryNamespace) { Seq( EntryWithItemIds( entityIdType = EntityIdType.Trends, sortIndex = module.sortIndex.get, size = module.items.size.toShort, itemIds = None ) ) } else { val moduleItems = module.items.map(_.item.id.asInstanceOf[Long]) val (entityId, items) = module.entryNamespace.toString match { case WhoToFollowCandidateDecorator.EntryNamespaceString => val itemIds = moduleItems.map(id => ItemIds(userId = Some(id))) (EntityIdType.WhoToFollow, itemIds) case WhoToSubscribeCandidateDecorator.EntryNamespaceString => val itemIds = moduleItems.map(id => ItemIds(userId = Some(id))) (EntityIdType.WhoToSubscribe, itemIds) case CommunitiesToJoinCandidateDecorator.EntryNamespaceString => val itemIds = moduleItems.map(id => ItemIds(communityId = Some(id))) (EntityIdType.CommunityModule, itemIds) case RecommendedJobsCandidateDecorator.EntryNamespaceString => val itemIds = moduleItems.map(id => ItemIds(jobId = Some(id))) (EntityIdType.JobModule, itemIds) case RecommendedRecruitingOrganizationsCandidateDecorator.EntryNamespaceString => val itemIds = moduleItems.map(id => ItemIds(recruitingOrganizationId = Some(id))) (EntityIdType.RecruitingOrganizationModule, itemIds) case BookmarksModuleCandidateDecorator.entryNamespaceString => (EntityIdType.BookmarksModule, Seq.empty) case PinnedTweetsModuleCandidateDecorator.entryNamespaceString => (EntityIdType.PinnedTweetsModule, Seq.empty) case VideoCarouselModuleCandidateDecorator.entryNamespaceString => (EntityIdType.VideoCarouselModule, Seq.empty) case EntryPointPivotModuleCandidateDecorator.EntryNamespaceString => (EntityIdType.EntryPointPivot, Seq.empty) case TuneFeedModuleCandidateDecorator.EntryNamespaceString => (EntityIdType.TuneFeedModule, Seq.empty) case other => throw new IllegalStateException("Invalid namespace: " + other) } val entry = EntryWithItemIds( entityIdType = entityId, sortIndex = module.sortIndex.get, size = module.items.size.toShort, itemIds = Some(items) ) Seq(entry) } }.flatten case ShowCoverInstruction(cover) => val entry = EntryWithItemIds( entityIdType = EntityIdType.Prompt, sortIndex = cover.sortIndex.get, size = 1, itemIds = None ) Seq(entry) case ReplaceEntryTimelineInstruction(replace) => val namespaceLength = TweetItem.TweetEntryNamespace.toString.length val itemId = ItemIds( tweetId = replace.entryIdToReplace.map(_.substring(namespaceLength + 1).toLong), entryIdToReplace = replace.entryIdToReplace, ) val entry = EntryWithItemIds( entityIdType = EntityIdType.Tweet, sortIndex = replace.sortIndex.get, size = 1, itemIds = Some(Seq(itemId)) ) Seq(entry) }.flatten val servedId = inputs.query.features.flatMap(_.getOrElse(ServedIdFeature, None)) val response = TimelineResponseV3( clientPlatform = timelineQuery.clientPlatform, servedTime = Time.now, requestType = requestTypeFromQuery(inputs.query), entries = entries, servedId = servedId, ) Stitch.callFuture(timelineResponseBatchesClient.insertResponse(timelineQuery, response)) } else Stitch.Unit } private def buildTweetEntryWithItemIds( candidate: ItemCandidateWithDetails, sortIndex: Long, debug: Boolean, promotedMetadata: Option[PromotedMetadata] = None ): EntryWithItemIds = { val features = candidate.features val sourceAuthorId = if (features.getOrElse(IsRetweetFeature, false)) features.getOrElse(SourceUserIdFeature, None) else features.getOrElse(AuthorIdFeature, None) val quoteAuthorId = if (features.getOrElse(QuotedTweetIdFeature, None).nonEmpty) features.getOrElse(SourceUserIdFeature, None) else None val tweetScore = features.getOrElse(ScoreFeature, None) val debugInfo = features.getOrElse(DebugStringFeature, None) val predictionRequestId = features.getOrElse(PredictionRequestIdFeature, None) val servedType = candidate.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined).name val isPreview = features.getOrElse(IsTweetPreviewFeature, default = false) val entityType = if (isPreview) EntityIdType.TweetPreview else EntityIdType.Tweet val grokAnnotations = if (debug) features.getOrElse(GrokAnnotationsFeature, None) else None val topics = grokAnnotations.map(_.topics) val tags = grokAnnotations.map(_.tags) val impressionId = if (debug) promotedMetadata.flatMap(_.impressionString) else None val predictedScores = if (debug) Some( PredictedScores( favoriteScore = features.getOrElse(PredictedFavoriteScoreFeature, None), replyScore = features.getOrElse(PredictedReplyScoreFeature, None), retweetScore = features.getOrElse(PredictedRetweetScoreFeature, None), replyEngagedByAuthorScore = features.getOrElse(PredictedReplyEngagedByAuthorScoreFeature, None), goodClickConvoDescFavoritedOrRepliedScore = features .getOrElse(PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, None), goodClickConvoDescUamGt2Score = features.getOrElse(PredictedGoodClickConvoDescUamGt2ScoreFeature, None), goodProfileClickScore = features.getOrElse(PredictedGoodProfileClickScoreFeature, None), videoQualityViewScore = features.getOrElse(PredictedVideoQualityViewScoreFeature, None), shareScore = features.getOrElse(PredictedShareScoreFeature, None), dwellScore = features.getOrElse(PredictedDwellScoreFeature, None), negativeFeedbackV2Score = features.getOrElse(PredictedNegativeFeedbackV2ScoreFeature, None) )) else None val itemId = ItemIds( tweetId = Some(candidate.candidateIdLong), sourceTweetId = features.getOrElse(SourceTweetIdFeature, None), quoteTweetId = features.getOrElse(QuotedTweetIdFeature, None), sourceAuthorId = sourceAuthorId, quoteAuthorId = quoteAuthorId, inReplyToTweetId = features.getOrElse(InReplyToTweetIdFeature, None), inReplyToAuthorId = features.getOrElse(DirectedAtUserIdFeature, None), semanticCoreId = features.getOrElse(SemanticCoreIdFeature, None), tweetScore = tweetScore.map( TweetScoreV1( _, Some(servedType), debugInfo, predictionRequestId, topics, tags, predictedScores) ), impressionId = impressionId ) EntryWithItemIds( entityIdType = entityType, sortIndex = sortIndex, size = 1.toShort, itemIds = Some(Seq(itemId)) ) } private def requestTypeFromQuery(query: PipelineQuery): persistence.RequestType = { val features = query.features.getOrElse(FeatureMap.empty) val featureToRequestType = Seq( (PollingFeature, persistence.RequestType.Polling), (GetInitialFeature, persistence.RequestType.Initial), (GetNewerFeature, persistence.RequestType.Newer), (GetMiddleFeature, persistence.RequestType.Middle), (GetOlderFeature, persistence.RequestType.Older) ) featureToRequestType .collectFirst { case (feature, requestType) if features.getOrElse(feature, false) => requestType }.getOrElse(persistence.RequestType.Other) } override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", ], exports = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/DeviceContextUnmarshaller.scala ================================================ package com.twitter.home_mixer.marshaller.request import com.twitter.home_mixer.model.request.DeviceContext import com.twitter.home_mixer.{thriftscala => t} import javax.inject.Inject import javax.inject.Singleton @Singleton class DeviceContextUnmarshaller @Inject() () { def apply(deviceContext: t.DeviceContext): DeviceContext = { DeviceContext( isPolling = deviceContext.isPolling, requestContext = deviceContext.requestContext, latestControlAvailable = deviceContext.latestControlAvailable, autoplayEnabled = deviceContext.autoplayEnabled ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerDebugParamsUnmarshaller.scala ================================================ package com.twitter.home_mixer.marshaller.request import com.twitter.home_mixer.model.request.HomeMixerDebugOptions import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.core.functional_component.marshaller.request.FeatureValueUnmarshaller import com.twitter.product_mixer.core.model.marshalling.request.DebugParams import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton @Singleton class HomeMixerDebugParamsUnmarshaller @Inject() ( featureValueUnmarshaller: FeatureValueUnmarshaller) { def apply(debugParams: t.DebugParams): DebugParams = { DebugParams( featureOverrides = debugParams.featureOverrides.map { map => map.mapValues(featureValueUnmarshaller(_)).toMap }, debugOptions = debugParams.debugOptions.map { options => HomeMixerDebugOptions( requestTimeOverride = options.requestTimeOverrideMillis.map(Time.fromMilliseconds), showIntermediateLogs = options.showIntermediateLogs.orElse(Some(false)) ) } ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductContextUnmarshaller.scala ================================================ package com.twitter.home_mixer.marshaller.request import com.twitter.home_mixer.model.request.FollowingProductContext import com.twitter.home_mixer.model.request.ForYouProductContext import com.twitter.home_mixer.model.request.HeavyRankerScoresProductContext import com.twitter.home_mixer.model.request.ScoredTweetsProductContext import com.twitter.home_mixer.model.request.ScoredVideoTweetsProductContext import com.twitter.home_mixer.model.request.SubscribedProductContext import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.core.model.marshalling.request.ProductContext import javax.inject.Inject import javax.inject.Singleton @Singleton class HomeMixerProductContextUnmarshaller @Inject() ( deviceContextUnmarshaller: DeviceContextUnmarshaller) { def apply(productContext: t.ProductContext): ProductContext = productContext match { case t.ProductContext.Following(p) => FollowingProductContext( deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), seenTweetIds = p.seenTweetIds, dspClientContext = p.dspClientContext ) case t.ProductContext.ForYou(p) => ForYouProductContext( deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), seenTweetIds = p.seenTweetIds, dspClientContext = p.dspClientContext ) case t.ProductContext.ListManagement(p) => throw new UnsupportedOperationException(s"This product is no longer used") case t.ProductContext.ScoredTweets(p) => ScoredTweetsProductContext( deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), seenTweetIds = p.seenTweetIds, servedTweetIds = p.servedTweetIds, backfillTweetIds = p.backfillTweetIds, signupCountryCode = p.signupCountryCode, allowForYouRecommendations = p.allowForYouRecommendations, signupSource = None, // not exposed in thrift interface followerCount = p.followerCount ) case t.ProductContext.ScoredVideoTweets(p) => ScoredVideoTweetsProductContext( deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), seenTweetIds = p.seenTweetIds, videoType = p.videoType, pinnedRelatedTweetIds = p.pinnedRelatedTweetIds, scorePinnedTweetsOnly = p.scorePinnedTweetsOnly, immersiveClientMetadata = p.immersiveClientMetadata ) case t.ProductContext.ListTweets(p) => throw new UnsupportedOperationException(s"This product is no longer used") case t.ProductContext.ListRecommendedUsers(p) => throw new UnsupportedOperationException(s"This product is no longer used") case t.ProductContext.Subscribed(p) => SubscribedProductContext( deviceContext = p.deviceContext.map(deviceContextUnmarshaller(_)), seenTweetIds = p.seenTweetIds, ) case t.ProductContext.HeavyRankerScores(p) => HeavyRankerScoresProductContext( deviceContext = p.tweetScoringRequestContext .flatMap(_.deviceContext.map(deviceContextUnmarshaller(_))), tweetIds = p.tweetIds ) case t.ProductContext.UnknownUnionField(field) => throw new UnsupportedOperationException(s"Unknown display context: ${field.field.name}") } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerProductUnmarshaller.scala ================================================ package com.twitter.home_mixer.marshaller.request import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.HeavyRankerScoresProduct import com.twitter.home_mixer.model.request.ScoredTweetsProduct import com.twitter.home_mixer.model.request.ScoredVideoTweetsProduct import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.core.model.marshalling.request.Product import javax.inject.Inject import javax.inject.Singleton @Singleton class HomeMixerProductUnmarshaller @Inject() () { def apply(product: t.Product): Product = product match { case t.Product.Following => FollowingProduct case t.Product.ForYou => ForYouProduct case t.Product.ListManagement => throw new UnsupportedOperationException(s"This product is no longer used") case t.Product.ScoredTweets => ScoredTweetsProduct case t.Product.ScoredVideoTweets => ScoredVideoTweetsProduct case t.Product.ListTweets => throw new UnsupportedOperationException(s"This product is no longer used") case t.Product.ListRecommendedUsers => throw new UnsupportedOperationException(s"This product is no longer used") case t.Product.Subscribed => SubscribedProduct case t.Product.HeavyRankerScores => HeavyRankerScoresProduct case t.Product.EnumUnknownProduct(value) => throw new UnsupportedOperationException(s"Unknown product: $value") } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request/HomeMixerRequestUnmarshaller.scala ================================================ package com.twitter.home_mixer.marshaller.request import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextUnmarshaller import javax.inject.Inject import javax.inject.Singleton @Singleton class HomeMixerRequestUnmarshaller @Inject() ( clientContextUnmarshaller: ClientContextUnmarshaller, homeProductUnmarshaller: HomeMixerProductUnmarshaller, homeProductContextUnmarshaller: HomeMixerProductContextUnmarshaller, homeDebugParamsUnmarshaller: HomeMixerDebugParamsUnmarshaller) { def apply(homeRequest: t.HomeMixerRequest): HomeMixerRequest = { HomeMixerRequest( clientContext = clientContextUnmarshaller(homeRequest.clientContext), product = homeProductUnmarshaller(homeRequest.product), productContext = homeRequest.productContext.map(homeProductContextUnmarshaller(_)), // Avoid de-serializing cursors in the request unmarshaller. The unmarshaller should never // fail, which is often a possibility when trying to de-serialize a cursor. Cursors can also // be product-specific and more appropriately handled in individual product pipelines. serializedRequestCursor = homeRequest.cursor, maxResults = homeRequest.maxResults, debugParams = homeRequest.debugParams.map(homeDebugParamsUnmarshaller(_)), homeRequestParam = false ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/PromotedTweetDetailsMarshaller.scala ================================================ package com.twitter.home_mixer.marshaller.timeline_logging import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog} object PromotedTweetDetailsMarshaller { def apply(entry: TweetItem, position: Int): thriftlog.PromotedTweetDetails = { thriftlog.PromotedTweetDetails( advertiserId = Some(entry.promotedMetadata.map(_.advertiserId).getOrElse(0L)), insertPosition = Some(position), impressionId = entry.promotedMetadata.flatMap(_.impressionString) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/TweetDetailsMarshaller.scala ================================================ package com.twitter.home_mixer.marshaller.timeline_logging import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.component_library.model.presentation.urt.UrtItemPresentation import com.twitter.product_mixer.component_library.model.presentation.urt.UrtModulePresentation import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.GeneralContextTypeMarshaller import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ConversationGeneralContextType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.GeneralContext import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContext import com.twitter.timelines.service.{thriftscala => tst} import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog} object TweetDetailsMarshaller { private val generalContextTypeMarshaller = new GeneralContextTypeMarshaller() def apply(entry: TweetItem, candidate: CandidateWithDetails): thriftlog.TweetDetails = { val socialContext = candidate.presentation.flatMap { case _ @UrtItemPresentation(timelineItem: TweetItem, _) => timelineItem.socialContext case _ @UrtModulePresentation(timelineModule) => timelineModule.items.head.item match { case timelineItem: TweetItem => timelineItem.socialContext case _ => Some(ConversationGeneralContextType) } } val socialContextType = socialContext match { case Some(GeneralContext(contextType, _, _, _, _)) => Some(generalContextTypeMarshaller(contextType).value.toShort) case Some(TopicContext(_, _)) => Some(tst.ContextType.Topic.value.toShort) case _ => None } thriftlog.TweetDetails( sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), socialContextType = socialContextType, suggestType = Some(candidate.features.get(ServedTypeFeature).name), authorId = candidate.features.getOrElse(AuthorIdFeature, None), sourceAuthorId = candidate.features.getOrElse(SourceUserIdFeature, None) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timeline_logging/WhoToFollowDetailsMarshaller.scala ================================================ package com.twitter.home_mixer.marshaller.timeline_logging import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem import com.twitter.timelines.timeline_logging.{thriftscala => thriftlog} object WhoToFollowDetailsMarshaller { def apply(entry: UserItem, candidate: ItemCandidateWithDetails): thriftlog.WhoToFollowDetails = thriftlog.WhoToFollowDetails( enableReactiveBlending = entry.enableReactiveBlending, impressionId = entry.promotedMetadata.flatMap(_.impressionString), advertiserId = entry.promotedMetadata.map(_.advertiserId) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", "src/thrift/com/twitter/timelineservice:thrift-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorMarshaller.scala ================================================ package com.twitter.home_mixer.marshaller.timelines import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor import com.twitter.timelines.service.{thriftscala => t} object ChronologicalCursorMarshaller { def apply(cursor: UrtOrderedCursor): Option[t.ChronologicalCursor] = { cursor.cursorType match { case Some(TopCursor) => Some(t.ChronologicalCursor(bottom = cursor.id)) case Some(BottomCursor) => Some(t.ChronologicalCursor(top = cursor.id)) case Some(GapCursor) => Some(t.ChronologicalCursor(top = cursor.id, bottom = cursor.gapBoundaryId)) case _ => None } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/ChronologicalCursorUnmarshaller.scala ================================================ package com.twitter.home_mixer.marshaller.timelines import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor import com.twitter.timelines.service.{thriftscala => t} object ChronologicalCursorUnmarshaller { def apply(requestCursor: t.RequestCursor): Option[UrtOrderedCursor] = { requestCursor match { case t.RequestCursor.ChronologicalCursor(cursor) => (cursor.top, cursor.bottom) match { case (Some(top), None) => Some(UrtOrderedCursor(top, cursor.top, Some(BottomCursor))) case (None, Some(bottom)) => Some(UrtOrderedCursor(bottom, cursor.bottom, Some(TopCursor))) case (Some(top), Some(bottom)) => Some(UrtOrderedCursor(top, cursor.top, Some(GapCursor), cursor.bottom)) case _ => None } case _ => None } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/DeviceContextMarshaller.scala ================================================ package com.twitter.home_mixer.marshaller.timelines import com.twitter.home_mixer.model.request.DeviceContext import com.twitter.product_mixer.core.model.marshalling.request.ClientContext import com.twitter.timelineservice.{thriftscala => t} import javax.inject.Inject import javax.inject.Singleton @Singleton class DeviceContextMarshaller @Inject() () { def apply(deviceContext: DeviceContext, clientContext: ClientContext): t.DeviceContext = { t.DeviceContext( countryCode = clientContext.countryCode, languageCode = clientContext.languageCode, clientAppId = clientContext.appId, ipAddress = clientContext.ipAddress, guestId = clientContext.guestId, userAgent = clientContext.userAgent, deviceId = clientContext.deviceId, isPolling = deviceContext.isPolling, requestContext = deviceContext.requestContext, referrer = None, tfeAuthHeader = None, mobileDeviceId = clientContext.mobileDeviceId, isSessionStart = None, latestControlAvailable = deviceContext.latestControlAvailable, guestIdMarketing = clientContext.guestIdMarketing, isInternalOrTwoffice = clientContext.isTwoffice, guestIdAds = clientContext.guestIdAds, isUrtRequest = Some(true) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/RecommendedUsersCursorUnmarshaller.scala ================================================ package com.twitter.home_mixer.marshaller.timelines import com.twitter.product_mixer.component_library.model.cursor.UrtUnorderedExcludeIdsCursor import com.twitter.timelines.service.{thriftscala => t} import com.twitter.util.Time object RecommendedUsersCursorUnmarshaller { def apply(requestCursor: t.RequestCursor): Option[UrtUnorderedExcludeIdsCursor] = { requestCursor match { case t.RequestCursor.RecommendedUsersCursor(cursor) => Some( UrtUnorderedExcludeIdsCursor( initialSortIndex = cursor.minSortIndex.getOrElse(Time.now.inMilliseconds), excludedIds = cursor.previouslyRecommendedUserIds )) case _ => None } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TimelineServiceCursorMarshaller.scala ================================================ package com.twitter.home_mixer.marshaller.timelines import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor import com.twitter.timelineservice.{thriftscala => t} object TimelineServiceCursorMarshaller { def apply(cursor: UrtOrderedCursor): Option[t.Cursor2] = { val id = cursor.id.map(_.toString) val gapBoundaryId = cursor.gapBoundaryId.map(_.toString) cursor.cursorType match { case Some(TopCursor) => Some(t.Cursor2(bottom = id)) case Some(BottomCursor) => Some(t.Cursor2(top = id)) case Some(GapCursor) => Some(t.Cursor2(top = id, bottom = gapBoundaryId)) case _ => None } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines/TopicContextFunctionalityTypeUnmarshaller.scala ================================================ package com.twitter.home_mixer.marshaller.timelines import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecWithEducationTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType import com.twitter.timelines.render.{thriftscala => urt} object TopicContextFunctionalityTypeUnmarshaller { def apply( topicContextFunctionalityType: urt.TopicContextFunctionalityType ): TopicContextFunctionalityType = topicContextFunctionalityType match { case urt.TopicContextFunctionalityType.Basic => BasicTopicContextFunctionalityType case urt.TopicContextFunctionalityType.Recommendation => RecommendationTopicContextFunctionalityType case urt.TopicContextFunctionalityType.RecWithEducation => RecWithEducationTopicContextFunctionalityType case urt.TopicContextFunctionalityType.EnumUnknownTopicContextFunctionalityType(field) => throw new UnsupportedOperationException(s"Unknown topic context functionality type: $field") } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/io/grpc:grpc-protobuf", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/signup", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", "search/search-router/thrift/src/main/thrift:thrift-scala", "src/java/com/twitter/ml/api:api-base", "src/java/com/twitter/ml/api/constant", "src/scala/com/twitter/timelines/prediction/features/common", "src/scala/com/twitter/timelines/prediction/features/conversation_features", "src/scala/com/twitter/timelines/prediction/features/engaged_language_features", "src/scala/com/twitter/timelines/prediction/features/recap", "src/scala/com/twitter/timelines/prediction/features/request_context", "src/scala/com/twitter/timelines/prediction/features/user_health", "src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", "strato/config/src/thrift/com/twitter/strato/columns/content_understanding:content_understanding-scala", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", "topic-social-proof/server/src/main/thrift:thrift-scala", "tweetconvosvc/common/src/main/thrift/com/twitter/tweetconvosvc/tweet_ancestor:thrift-scala", "user_history_transformer/service/src/main/java/com/x/user_action_sequence", ], exports = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/signup", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", "tweetconvosvc/common/src/main/thrift/com/twitter/tweetconvosvc/tweet_ancestor:thrift-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ClearCacheIncludeInstruction.scala ================================================ package com.twitter.home_mixer.model import com.twitter.home_mixer.model.request.DeviceContext.RequestContext import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.product_mixer.component_library.premarshaller.urt.builder.IncludeInstruction import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam /** * Include a clear cache timeline instruction when we satisfy these criteria: * - Request Provenance * - At least N non-ad tweet entries in the response * * This is to ensure that we have sufficient new content to justify jumping users to the * top of the new timelines response and don't add unnecessary load to backend systems */ case class ClearCacheIncludeInstruction( ptrEnableParam: FSParam[Boolean], coldStartEnableParam: FSParam[Boolean], warmStartEnableParam: FSParam[Boolean], manualRefreshEnableParam: FSParam[Boolean], navigateEnableParam: FSParam[Boolean], minEntriesParam: FSBoundedParam[Int]) extends IncludeInstruction[PipelineQuery with HasDeviceContext] { override def apply( query: PipelineQuery with HasDeviceContext, entries: Seq[TimelineEntry] ): Boolean = { val requestContext = query.deviceContext.flatMap(_.requestContextValue) val ptrEnabled = query.params(ptrEnableParam) && requestContext.contains(RequestContext.PullToRefresh) val coldStartEnabled = query.params(coldStartEnableParam) && requestContext.contains(RequestContext.Launch) val warmStartEnabled = query.params(warmStartEnableParam) && requestContext.contains(RequestContext.Foreground) val manualRefreshEnabled = query.params(manualRefreshEnableParam) && requestContext.contains( RequestContext.ManualRefresh) val navigateEnabled = query.params(navigateEnableParam) && requestContext.contains(RequestContext.Navigate) val minTweets = query.params(minEntriesParam) <= entries.collect { case item: TweetItem if item.promotedMetadata.isEmpty => 1 case module: TimelineModule if module.items.head.item.isInstanceOf[TweetItem] => module.items.size }.sum (ptrEnabled || coldStartEnabled || warmStartEnabled || manualRefreshEnabled || navigateEnabled) && minTweets } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/ContentFeatures.scala ================================================ package com.twitter.home_mixer.model import com.twitter.escherbird.{thriftscala => esb} import com.twitter.search.common.features.{thriftscala => sc} import com.twitter.tweetypie.{thriftscala => tp} object ContentFeatures { val Empty: ContentFeatures = ContentFeatures( 0.toShort, false, 0.toShort, 0.toShort, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None ) def fromThrift(ebFeatures: sc.ThriftTweetFeatures): ContentFeatures = ContentFeatures( length = ebFeatures.tweetLength.getOrElse(0).toShort, hasQuestion = ebFeatures.hasQuestion.getOrElse(false), numCaps = ebFeatures.numCaps.getOrElse(0).toShort, numWhiteSpaces = ebFeatures.numWhitespaces.getOrElse(0).toShort, numNewlines = ebFeatures.numNewlines, videoDurationMs = ebFeatures.videoDurationMs, bitRate = ebFeatures.bitRate, aspectRatioNum = ebFeatures.aspectRatioNum, aspectRatioDen = ebFeatures.aspectRatioDen, widths = ebFeatures.widths.map(_.map(_.toShort)), heights = ebFeatures.heights.map(_.map(_.toShort)), resizeMethods = ebFeatures.resizeMethods.map(_.map(_.toShort)), numMediaTags = ebFeatures.numMediaTags.map(_.toShort), mediaTagScreenNames = ebFeatures.mediaTagScreenNames, emojiTokens = ebFeatures.emojiTokens.map(_.toSet), emoticonTokens = ebFeatures.emoticonTokens.map(_.toSet), faceAreas = ebFeatures.faceAreas, dominantColorRed = ebFeatures.dominantColorRed, dominantColorBlue = ebFeatures.dominantColorBlue, dominantColorGreen = ebFeatures.dominantColorGreen, numColors = ebFeatures.numColors.map(_.toShort), stickerIds = ebFeatures.stickerIds, mediaOriginProviders = ebFeatures.mediaOriginProviders, isManaged = ebFeatures.isManaged, is360 = ebFeatures.is360, viewCount = ebFeatures.viewCount, isMonetizable = ebFeatures.isMonetizable, isEmbeddable = ebFeatures.isEmbeddable, hasSelectedPreviewImage = ebFeatures.hasSelectedPreviewImage, hasTitle = ebFeatures.hasTitle, hasDescription = ebFeatures.hasDescription, hasVisitSiteCallToAction = ebFeatures.hasVisitSiteCallToAction, hasAppInstallCallToAction = ebFeatures.hasAppInstallCallToAction, hasWatchNowCallToAction = ebFeatures.hasWatchNowCallToAction, dominantColorPercentage = ebFeatures.dominantColorPercentage, posUnigrams = ebFeatures.posUnigrams.map(_.toSet), posBigrams = ebFeatures.posBigrams.map(_.toSet), semanticCoreAnnotations = ebFeatures.semanticCoreAnnotations, tokens = ebFeatures.textTokens.map(_.toSeq), conversationControl = ebFeatures.conversationControl, // media and selfThreadMetadata not carried by ThriftTweetFeatures media = None, selfThreadMetadata = None, hasImage = None, hasVideo = None ) } case class ContentFeatures( length: Short, hasQuestion: Boolean, numCaps: Short, numWhiteSpaces: Short, numNewlines: Option[Short], videoDurationMs: Option[Int], bitRate: Option[Int], aspectRatioNum: Option[Short], aspectRatioDen: Option[Short], widths: Option[Seq[Short]], heights: Option[Seq[Short]], resizeMethods: Option[Seq[Short]], numMediaTags: Option[Short], mediaTagScreenNames: Option[Seq[String]], emojiTokens: Option[Set[String]], emoticonTokens: Option[Set[String]], faceAreas: Option[Seq[Int]], dominantColorRed: Option[Short], dominantColorBlue: Option[Short], dominantColorGreen: Option[Short], numColors: Option[Short], stickerIds: Option[Seq[Long]], mediaOriginProviders: Option[Seq[String]], isManaged: Option[Boolean], is360: Option[Boolean], viewCount: Option[Long], isMonetizable: Option[Boolean], isEmbeddable: Option[Boolean], hasSelectedPreviewImage: Option[Boolean], hasTitle: Option[Boolean], hasDescription: Option[Boolean], hasVisitSiteCallToAction: Option[Boolean], hasAppInstallCallToAction: Option[Boolean], hasWatchNowCallToAction: Option[Boolean], media: Option[Seq[tp.MediaEntity]], dominantColorPercentage: Option[Double], posUnigrams: Option[Set[String]], posBigrams: Option[Set[String]], semanticCoreAnnotations: Option[Seq[esb.TweetEntityAnnotation]], selfThreadMetadata: Option[tp.SelfThreadMetadata], tokens: Option[Seq[String]], conversationControl: Option[tp.ConversationControl], hasImage: Option[Boolean], hasVideo: Option[Boolean], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/GapIncludeInstruction.scala ================================================ package com.twitter.home_mixer.model import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdBottomTweetFeature import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdResponseTruncatedFeature import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.component_library.premarshaller.urt.builder.IncludeInstruction import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor import com.twitter.product_mixer.core.pipeline.HasPipelineCursor import com.twitter.product_mixer.core.pipeline.PipelineQuery /** * Determine whether to include a Gap Cursor in the response based on whether a timeline * is truncated because it has more entries than the max response size. * There are two ways this can happen: * 1) There are unused entries in Earlybird. This is determined by a flag returned from Earlybird. * We respect the Earlybird flag only if there are some entries after deduping and filtering * to ensure that we do not get stuck repeatedly serving gaps which lead to no tweets. * 2) Ads injection can take the response size over the max count. Goldfinch truncates tweet * entries in this case. We can check if the bottom tweet from Earlybird is in the response to * determine if all Earlybird tweets have been used. * * While scrolling down to get older tweets (BottomCursor), responses will generally be * truncated, but we don't want to render a gap cursor there, so we need to ensure we only * apply the truncation check to newer (TopCursor) or middle (GapCursor) requests. * * We return either a Gap Cursor or a Bottom Cursor, but not both, so the include instruction * for Bottom should be the inverse of Gap. */ object GapIncludeInstruction extends IncludeInstruction[PipelineQuery with HasPipelineCursor[UrtOrderedCursor]] { override def apply( query: PipelineQuery with HasPipelineCursor[UrtOrderedCursor], entries: Seq[TimelineEntry] ): Boolean = { val wasTruncated = query.features.exists(_.getOrElse(EarlybirdResponseTruncatedFeature, false)) // Get oldest tweet or tweets within oldest conversation module val tweetEntries = entries.view.reverse .collectFirst { case item: TweetItem if item.promotedMetadata.isEmpty => Seq(item.id.toString) case module: TimelineModule if module.items.head.item.isInstanceOf[TweetItem] => module.items.map(_.item.id.toString) }.toSeq.flatten val bottomCursor = query.features.flatMap(_.getOrElse(EarlybirdBottomTweetFeature, None)).map(_.toString) // Ads truncation happened if we have at least max count entries and bottom tweet is missing val adsTruncation = query.requestedMaxResults.exists(_ <= entries.size) && !bottomCursor.exists(tweetEntries.contains) query.pipelineCursor.exists(_.cursorType match { case Some(TopCursor) | Some(GapCursor) => (wasTruncated && tweetEntries.nonEmpty) || adsTruncation case _ => false }) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/GrokTopics.scala ================================================ package com.twitter.home_mixer.model object GrokTopics { val GrokCategoryIdToNameMap = Map( -> "Sports", -> "Anime", -> "Celebrity", -> "Music", -> "News", -> "Business & Finance", -> "Cryptocurrency", -> "Technology", -> "Science", -> "Gaming", -> "Movies & TV", -> "Travel", -> "Food", -> "Health & Fitness", -> "Memes", -> "Art", -> "Fashion", -> "Religion", -> "Shopping", -> "Cars", -> "Aviation", -> "Motorcycles", -> "Beauty", -> "Nature & Outdoors", -> "Pets", -> "Relationships", -> "Home & Garden", -> "Career", -> "Dance", -> "Education", -> "Podcasts", -> "Streaming" ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeAdsQuery.scala ================================================ package com.twitter.home_mixer.model import com.twitter.adserver.thriftscala.RequestTriggerType import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature import com.twitter.home_mixer.model.HomeFeatures.PollingFeature import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.component_library.model.query.ads.AdsQuery import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.pipeline.HasPipelineCursor import com.twitter.product_mixer.core.pipeline.PipelineQuery /** * These are for feeds needed for ads only. */ trait HomeAdsQuery extends AdsQuery with PipelineQuery with HasDeviceContext with HasPipelineCursor[UrtOrderedCursor] { private val featureToRequestTriggerType = Seq( (GetInitialFeature, RequestTriggerType.Initial), (GetNewerFeature, RequestTriggerType.Scroll), (GetOlderFeature, RequestTriggerType.Scroll), (PollingFeature, RequestTriggerType.AutoRefresh) ) override val autoplayEnabled: Option[Boolean] = deviceContext.flatMap(_.autoplayEnabled) override def requestTriggerType: Option[RequestTriggerType] = { val features = this.features.getOrElse(FeatureMap.empty) featureToRequestTriggerType.collectFirst { case (feature, requestType) if features.get(feature) => Some(requestType) }.flatten } override val disableNsfwAvoidance: Option[Boolean] = Some(true) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeFeatures.scala ================================================ package com.twitter.home_mixer.model import com.twitter.core_workflows.user_model.{thriftscala => um} import com.twitter.dal.personal_data.thriftjava.PersonalDataType import com.twitter.dal.personal_data.{thriftjava => pd} import com.twitter.gizmoduck.{thriftscala => gt} import com.twitter.home_mixer.model.candidate_source.SourceSignal import com.twitter.home_mixer.model.signup.SignupSource import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.timelines.render.{thriftscala => urt} import com.twitter.mediaservices.commons.thriftscala.MediaCategory import com.twitter.ml.api.constant.SharedFeatures import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.BoolDataRecordCompatible import com.twitter.product_mixer.core.feature.datarecord.DataRecordFeature import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible import com.twitter.product_mixer.core.feature.datarecord.LongDiscreteDataRecordCompatible import com.twitter.product_mixer.core.feature.datarecord.SparseBinaryDataRecordCompatible import com.twitter.product_mixer.core.feature.datarecord.SparseContinuousDataRecordCompatible import com.twitter.product_mixer.core.feature.datarecord.StringDataRecordCompatible import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.ModuleDisplayType import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.search.common.features.{thriftscala => sc} import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.strato.columns.content_understanding.thriftscala.AnnotatedVideoDeserialized import com.twitter.timelinemixer.clients.manhattan.DismissInfo import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 import com.twitter.timelinemixer.injection.model.candidate.AudioSpaceMetaData import com.twitter.timelines.conversation_features.v1.thriftscala.ConversationFeatures import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} import com.twitter.timelines.model.UserId import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import com.twitter.timelines.prediction.features.conversation_features import com.twitter.timelines.prediction.features.engaged_language_features.LanguageFeatures import com.twitter.timelines.prediction.features.engagement_features.EngagementDataRecordFeatures import com.twitter.timelines.prediction.features.recap.RecapFeatures import com.twitter.timelines.prediction.features.request_context.RequestContextFeatures import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures import com.twitter.timelineservice.model.FeedbackEntry import com.twitter.timelineservice.suggests.{thriftscala => st} import com.twitter.tsp.{thriftscala => tsp} import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} import com.twitter.util.Duration import com.twitter.util.Time import com.x.user_action_sequence.UserActionSequence object HomeFeatures { // Candidate Features object AncestorsFeature extends Feature[TweetCandidate, Seq[ta.TweetAncestor]] object AudioSpaceMetaDataFeature extends Feature[TweetCandidate, Option[AudioSpaceMetaData]] object ListIdFeature extends Feature[TweetCandidate, Option[Long]] object ListNameFeature extends Feature[TweetCandidate, Option[String]] object ValidLikedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] object BookmarkedTweetTimestamp extends Feature[TweetCandidate, Option[Long]] object ArticleIdFeature extends Feature[TweetCandidate, Option[Long]] object ArticlePreviewTextFeature extends Feature[TweetCandidate, Option[String]] /** * For Retweets, this should refer to the retweeting user. Use [[SourceUserIdFeature]] if you want to know * who created the Tweet that was retweeted. */ object AuthorIdFeature extends DataRecordOptionalFeature[TweetCandidate, Long] with LongDiscreteDataRecordCompatible { override val featureName: String = SharedFeatures.AUTHOR_ID.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.UserId) } object AuthorAccountAge extends Feature[TweetCandidate, Option[Duration]] object AuthorIsBlueVerifiedFeature extends Feature[TweetCandidate, Boolean] object AuthorIsGoldVerifiedFeature extends Feature[TweetCandidate, Boolean] object AuthorIsGrayVerifiedFeature extends Feature[TweetCandidate, Boolean] object AuthorIsLegacyVerifiedFeature extends Feature[TweetCandidate, Boolean] object AuthorIsCreatorFeature extends Feature[TweetCandidate, Boolean] object AuthorIsProtectedFeature extends Feature[TweetCandidate, Boolean] object AuthorFollowersFeature extends Feature[TweetCandidate, Option[Long]] object ViralContentCreatorFeature extends Feature[TweetCandidate, Boolean] object GrokContentCreatorFeature extends Feature[TweetCandidate, Boolean] object GorkContentCreatorFeature extends Feature[TweetCandidate, Boolean] object AuthoredByContextualUserFeature extends Feature[TweetCandidate, Boolean] object CachedCandidatePipelineIdentifierFeature extends Feature[TweetCandidate, Option[String]] object ConversationFeature extends Feature[TweetCandidate, Option[ConversationFeatures]] object AuthorSafetyLabels extends Feature[TweetCandidate, Option[Seq[String]]] /** * This field should be set to the focal Tweet's tweetId for all tweets which are expected to * be rendered in the same convo module. For non-convo module Tweets, this will be * set to None. Note this is different from how TweetyPie defines ConversationId which is defined * on all Tweets and points to the root tweet. This feature is used for grouping convo modules together. */ object ConversationModuleFocalTweetIdFeature extends Feature[TweetCandidate, Option[Long]] /** * This field should always be set to the root Tweet in a conversation for all Tweets. For replies, this will * point back to the root Tweet. For non-replies, this will be the candidate's Tweet id. This is consistent with * the TweetyPie definition of ConversationModuleId. */ object ConversationModuleIdFeature extends Feature[TweetCandidate, Option[Long]] object DirectedAtUserIdFeature extends Feature[TweetCandidate, Option[Long]] object EarlybirdFeature extends Feature[TweetCandidate, Option[sc.ThriftTweetFeatures]] object EarlybirdScoreFeature extends Feature[TweetCandidate, Option[Double]] object EarlybirdSearchResultFeature extends Feature[TweetCandidate, Option[eb.ThriftSearchResult]] object EntityTokenFeature extends Feature[TweetCandidate, Option[String]] object ExclusiveConversationAuthorIdFeature extends Feature[TweetCandidate, Option[Long]] object FavoritedByCountFeature extends DataRecordFeature[TweetCandidate, Double] with DoubleDataRecordCompatible { override val featureName: String = EngagementDataRecordFeatures.InNetworkFavoritesCount.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.CountOfPrivateLikes, pd.PersonalDataType.CountOfPublicLikes) } object FavoritedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] object FeedbackHistoryFeature extends Feature[TweetCandidate, Seq[FeedbackEntry]] // A boolean feature indicating whether the latest feedback timestamp till now is newer than the cached scored tweets TTL object HasRecentFeedbackSinceCacheTtlFeature extends Feature[PipelineQuery, Boolean] object SlopAuthorFeature extends Feature[TweetCandidate, Boolean] object SlopAuthorScoreFeature extends DataRecordOptionalFeature[TweetCandidate, Double] with DoubleDataRecordCompatible { override val featureName: String = RecapFeatures.SLOP_AUTHOR_SCORE.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object GrokTranslatedPostIsCachedFeature extends Feature[TweetCandidate, Boolean] object GrokVideoMetadataFeature extends FeatureWithDefaultOnFailure[TweetCandidate, Option[AnnotatedVideoDeserialized]] { override def defaultValue: Option[AnnotatedVideoDeserialized] = None } object GrokCategoryDataRecordFeature extends DataRecordOptionalFeature[TweetCandidate, Map[String, Double]] with SparseContinuousDataRecordCompatible { override val featureName: String = RecapFeatures.GROK_CATEGORY_SCORES.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.InferredInterests) } object GrokTagsFeature extends Feature[TweetCandidate, Set[String]] with SparseBinaryDataRecordCompatible { override val featureName: String = RecapFeatures.GROK_TAGS.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set( pd.PersonalDataType.InferredLanguage) } object GrokIsGoreFeature extends DataRecordOptionalFeature[TweetCandidate, Boolean] with BoolDataRecordCompatible { override val featureName: String = RecapFeatures.GROK_IS_GORE.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object GrokIsNsfwFeature extends DataRecordOptionalFeature[TweetCandidate, Boolean] with BoolDataRecordCompatible { override val featureName: String = RecapFeatures.GROK_IS_NSFW.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object GrokIsSoftNsfwFeature extends DataRecordOptionalFeature[TweetCandidate, Boolean] with BoolDataRecordCompatible { override val featureName: String = RecapFeatures.GROK_IS_SOFT_NSFW.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object GrokIsSpamFeature extends DataRecordOptionalFeature[TweetCandidate, Boolean] with BoolDataRecordCompatible { override val featureName: String = RecapFeatures.GROK_IS_SPAM.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object GrokIsViolentFeature extends DataRecordOptionalFeature[TweetCandidate, Boolean] with BoolDataRecordCompatible { override val featureName: String = RecapFeatures.GROK_IS_VIOLENT.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object GrokIsLowQualityFeature extends DataRecordOptionalFeature[TweetCandidate, Boolean] with BoolDataRecordCompatible { override val featureName: String = RecapFeatures.GROK_IS_LOW_QUALITY.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.InferredInterests) } object GrokIsOcrFeature extends DataRecordOptionalFeature[TweetCandidate, Boolean] with BoolDataRecordCompatible { override val featureName: String = RecapFeatures.GROK_IS_OCR.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.InferredInterests) } object GrokSunnyScoreFeature extends DataRecordOptionalFeature[TweetCandidate, Double] with DoubleDataRecordCompatible { override val featureName: String = RecapFeatures.GROK_SUNNY_SCORE.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set( pd.PersonalDataType.EngagementScore) } // Used only for metrics tracking. Does not affect the recommendations. object GrokPoliticalInclinationFeature extends Feature[TweetCandidate, Option[hmt.PoliticalInclination]] object GrokSlopScoreFeature extends Feature[TweetCandidate, Option[Long]] object DedupClusterIdFeature extends Feature[TweetCandidate, Option[Long]] object DedupClusterId88Feature extends Feature[TweetCandidate, Option[Long]] object RetweetedByCountFeature extends DataRecordFeature[TweetCandidate, Double] with DoubleDataRecordCompatible { override val featureName: String = EngagementDataRecordFeatures.InNetworkRetweetsCount.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.CountOfPrivateRetweets, pd.PersonalDataType.CountOfPublicRetweets) } object RetweetedByEngagerIdsFeature extends Feature[TweetCandidate, Seq[Long]] object RepliedByCountFeature extends DataRecordFeature[TweetCandidate, Double] with DoubleDataRecordCompatible { override val featureName: String = EngagementDataRecordFeatures.InNetworkRepliesCount.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.CountOfPrivateReplies, pd.PersonalDataType.CountOfPublicReplies) } object RepliedByEngagerIdsFeature extends Feature[TweetCandidate, Seq[Long]] object FollowedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] object TopicIdSocialContextFeature extends Feature[TweetCandidate, Option[Long]] object TopicContextFunctionalityTypeFeature extends Feature[TweetCandidate, Option[TopicContextFunctionalityType]] object BasketballContextFeature extends Feature[TweetCandidate, Option[urt.BasketballContext]] object GenericPostContextFeature extends Feature[TweetCandidate, Option[urt.GenericContext]] object FromInNetworkSourceFeature extends Feature[TweetCandidate, Boolean] object FullScoringSucceededFeature extends Feature[TweetCandidate, Boolean] object HasDisplayedTextFeature extends Feature[TweetCandidate, Boolean] object InReplyToTweetIdFeature extends Feature[TweetCandidate, Option[Long]] object InReplyToUserIdFeature extends Feature[TweetCandidate, Option[Long]] object IsArticleFeature extends Feature[TweetCandidate, Boolean] object IsAncestorCandidateFeature extends Feature[TweetCandidate, Boolean] object IsBoostedCandidateFeature extends Feature[TweetCandidate, Boolean] object IsExtendedReplyFeature extends DataRecordFeature[TweetCandidate, Boolean] with BoolDataRecordCompatible { override val featureName: String = RecapFeatures.IS_EXTENDED_REPLY.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object IsInReplyToReplyOrDirectedFeature extends Feature[TweetCandidate, Boolean] object IsInReplyToRetweetFeature extends Feature[TweetCandidate, Boolean] object IsRandomTweetFeature extends DataRecordFeature[TweetCandidate, Boolean] with BoolDataRecordCompatible { override val featureName: String = TimelinesSharedFeatures.IS_RANDOM_TWEET.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object IsReadFromCacheFeature extends Feature[TweetCandidate, Boolean] object IsRetweetFeature extends Feature[TweetCandidate, Boolean] object IsRetweetedReplyFeature extends Feature[TweetCandidate, Boolean] object IsSupportAccountReplyFeature extends Feature[TweetCandidate, Boolean] object LastScoredTimestampMsFeature extends Feature[TweetCandidate, Option[Long]] object NonSelfFavoritedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] object NumImagesFeature extends Feature[TweetCandidate, Option[Int]] object PositionFeature extends Feature[TweetCandidate, Option[Int]] // Internal id generated per prediction service request object PredictionRequestIdFeature extends Feature[TweetCandidate, Option[Long]] object QuotedTweetIdFeature extends Feature[TweetCandidate, Option[Long]] object QuotedUserIdFeature extends Feature[TweetCandidate, Option[Long]] object PhoenixScoreFeature extends Feature[TweetCandidate, Option[Double]] object ScoreFeature extends Feature[TweetCandidate, Option[Double]] object IsColdStartPostFeature extends Feature[TweetCandidate, Boolean] object SemanticCoreIdFeature extends Feature[TweetCandidate, Option[Long]] object SimclustersTweetTopKClustersWithScoresFeature extends Feature[TweetCandidate, Map[String, Double]] // Tweet ID of the source tweet if the candidate is a retweet object SourceTweetIdFeature extends Feature[TweetCandidate, Option[Long]] // Tweet ID of the source tweet if the candidate is a retweet. Tweet id of the candidate otherwise object OriginalTweetIdFeature extends DataRecordOptionalFeature[TweetCandidate, Long] with LongDiscreteDataRecordCompatible { override val featureName: String = TimelinesSharedFeatures.SOURCE_TWEET_ID.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.TweetId) } object SourceUserIdFeature extends Feature[TweetCandidate, Option[Long]] object ServedTypeFeature extends Feature[TweetCandidate, hmt.ServedType] object TSPMetricTagFeature extends Feature[TweetCandidate, Set[tsp.MetricTag]] object TweetLanguageFeature extends Feature[TweetCandidate, Option[String]] object TweetMediaIdsFeature extends Feature[TweetCandidate, Seq[Long]] object TweetMediaClusterIdsFeature extends Feature[TweetCandidate, Map[Long, Long]] object TweetMediaCompletionRateFeature extends Feature[TweetCandidate, Option[Double]] object ClipImageClusterIdsFeature extends Feature[TweetCandidate, Map[Long, Long]] object MultiModalEmbeddingsFeature extends Feature[TweetCandidate, Option[Seq[Double]]] object FirstMediaIdFeature extends DataRecordOptionalFeature[TweetCandidate, Long] with LongDiscreteDataRecordCompatible { override val featureName: String = TimelinesSharedFeatures.FIRST_MEDIA_ID.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.TweetId) } object TweetUrlsFeature extends Feature[TweetCandidate, Seq[String]] object ViewCountFeature extends Feature[TweetCandidate, Option[Long]] object VideoDurationMsFeature extends Feature[TweetCandidate, Option[Int]] object ViewerIdFeature extends DataRecordFeature[TweetCandidate, Long] with LongDiscreteDataRecordCompatible { override def featureName: String = SharedFeatures.USER_ID.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.UserId) } object WeightedModelScoreFeature extends Feature[TweetCandidate, Option[Double]] object MentionUserIdFeature extends Feature[TweetCandidate, Seq[Long]] object MentionScreenNameFeature extends Feature[TweetCandidate, Seq[String]] object HasImageFeature extends Feature[TweetCandidate, Boolean] object HasVideoFeature extends Feature[TweetCandidate, Boolean] object VideoAspectRatioFeature extends Feature[TweetCandidate, Option[Float]] object VideoDisplayTypeFeature extends Feature[TweetCandidate, Option[ModuleDisplayType]] object VideoHeightFeature extends Feature[TweetCandidate, Option[Short]] object VideoWidthFeature extends Feature[TweetCandidate, Option[Short]] object MediaIdFeature extends Feature[TweetCandidate, Option[Long]] object HasMultipleMedia extends Feature[TweetCandidate, Boolean] object MediaCategoryFeature extends Feature[TweetCandidate, Option[MediaCategory]] object SemanticAnnotationIdsFeature extends Feature[TweetCandidate, Seq[Long]] object TweetTypeMetricsFeature extends Feature[TweetCandidate, Option[Seq[Byte]]] object CurrentPinnedTweetFeature extends Feature[TweetCandidate, Option[Long]] // Tweetypie VF Features object IsHydratedFeature extends Feature[TweetCandidate, Boolean] object OonNsfwFeature extends Feature[TweetCandidate, Boolean] object QuotedTweetDroppedFeature extends Feature[TweetCandidate, Boolean] // Raw Tweet Text from Tweetypie object TweetTextFeature extends Feature[TweetCandidate, Option[String]] object TweetTextTokensFeature extends Feature[TweetCandidate, Option[Seq[Int]]] object AuthorEnabledPreviewsFeature extends Feature[TweetCandidate, Boolean] object IsTweetPreviewFeature extends Feature[TweetCandidate, Boolean] // SGS Features /** * By convention, this is set to true for retweets of non-followed authors * E.g. where somebody the viewer follows retweets a Tweet from somebody the viewer doesn't follow */ object InNetworkFeature extends FeatureWithDefaultOnFailure[TweetCandidate, Boolean] { override val defaultValue: Boolean = true } // Query Features object AccountAgeFeature extends Feature[PipelineQuery, Option[Time]] object ClientIdFeature extends DataRecordOptionalFeature[PipelineQuery, Long] with LongDiscreteDataRecordCompatible { override def featureName: String = SharedFeatures.CLIENT_ID.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.ClientType) } object CachedScoredTweetsFeature extends Feature[PipelineQuery, Seq[hmt.ScoredTweet]] object FollowsSportsAccountFeature extends Feature[PipelineQuery, Boolean] object DeviceCountryFeature extends DataRecordOptionalFeature[PipelineQuery, String] with StringDataRecordCompatible { override def featureName: String = RequestContextFeatures.COUNTRY_CODE.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.PrivateCountryOrRegion, pd.PersonalDataType.InferredCountry) } object DeviceLanguageFeature extends DataRecordOptionalFeature[PipelineQuery, String] with StringDataRecordCompatible { override def featureName: String = RequestContextFeatures.LANGUAGE_CODE.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set( pd.PersonalDataType.GeneralSettings, pd.PersonalDataType.ProvidedLanguage, pd.PersonalDataType.InferredLanguage) } object UuaUserGenderFeature extends DataRecordOptionalFeature[PipelineQuery, String] with StringDataRecordCompatible { override def featureName: String = UserHealthFeatures.UserGender.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set( pd.PersonalDataType.GeneralSettings, pd.PersonalDataType.ProvidedGender, pd.PersonalDataType.InferredGender) } object UuaUserStateFeature extends DataRecordOptionalFeature[PipelineQuery, Long] with LongDiscreteDataRecordCompatible { override def featureName: String = UserHealthFeatures.UserState.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set( pd.PersonalDataType.UserState, pd.PersonalDataType.UserType ) } object UuaUserAgeBucketFeature extends DataRecordOptionalFeature[PipelineQuery, String] with StringDataRecordCompatible { override def featureName: String = UserHealthFeatures.UserAgeBucket.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set( pd.PersonalDataType.GeneralSettings, pd.PersonalDataType.ProvidedAge, pd.PersonalDataType.InferredAge ) } object DismissInfoFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Map[st.SuggestType, Option[DismissInfo]]] { override def defaultValue: Map[st.SuggestType, Option[DismissInfo]] = Map.empty } object FollowingLastNonPollingTimeFeature extends Feature[PipelineQuery, Option[Time]] object GetInitialFeature extends DataRecordFeature[PipelineQuery, Boolean] with BoolDataRecordCompatible { override def featureName: String = RequestContextFeatures.IS_GET_INITIAL.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object GetMiddleFeature extends DataRecordFeature[PipelineQuery, Boolean] with BoolDataRecordCompatible { override def featureName: String = RequestContextFeatures.IS_GET_MIDDLE.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object GetNewerFeature extends DataRecordFeature[PipelineQuery, Boolean] with BoolDataRecordCompatible { override def featureName: String = RequestContextFeatures.IS_GET_NEWER.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object GetOlderFeature extends DataRecordFeature[PipelineQuery, Boolean] with BoolDataRecordCompatible { override def featureName: String = RequestContextFeatures.IS_GET_OLDER.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object GuestIdFeature extends DataRecordOptionalFeature[PipelineQuery, Long] with LongDiscreteDataRecordCompatible { override def featureName: String = SharedFeatures.GUEST_ID.getFeatureName override def personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.GuestId) } object GrokAnnotationsFeature extends Feature[TweetCandidate, Option[hmt.GrokAnnotations]] object GrokTopCategoryFeature extends Feature[TweetCandidate, Option[Long]] object HasDarkRequestFeature extends Feature[PipelineQuery, Option[Boolean]] object ImpressionBloomFilterFeature extends FeatureWithDefaultOnFailure[PipelineQuery, blm.ImpressionBloomFilterSeq] { override def defaultValue: blm.ImpressionBloomFilterSeq = blm.ImpressionBloomFilterSeq(Seq.empty) } object ImpressedMediaIds extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { override val defaultValue: Seq[Long] = Seq.empty } object IsForegroundRequestFeature extends Feature[PipelineQuery, Boolean] object IsLaunchRequestFeature extends Feature[PipelineQuery, Boolean] object LastNonPollingTimeFeature extends Feature[PipelineQuery, Option[Time]] object LastNegativeFeedbackTimeFeature extends Feature[PipelineQuery, Option[Time]] object LowSignalUserFeature extends Feature[PipelineQuery, Boolean] object NaviClientConfigFeature extends Feature[PipelineQuery, NaviClientConfig] object NonPollingTimesFeature extends Feature[PipelineQuery, Seq[Long]] object PersistenceEntriesFeature extends Feature[PipelineQuery, Seq[TimelineResponseV3]] object PollingFeature extends Feature[PipelineQuery, Boolean] object PullToRefreshFeature extends Feature[PipelineQuery, Boolean] // Scores from Real Graph representing the relationship between the viewer and another user object RealGraphInNetworkScoresFeature extends Feature[PipelineQuery, Map[UserId, Double]] object ImmersiveClientEmbeddingsFeature extends Feature[PipelineQuery, Map[Long, Seq[Double]]] object RequestJoinIdFeature extends Feature[TweetCandidate, Option[Long]] // Internal id generated per request object ServedIdFeature extends Feature[PipelineQuery, Option[Long]] object ServedTweetIdsFeature extends Feature[PipelineQuery, Seq[Long]] object ServedAuthorIdsFeature extends Feature[PipelineQuery, Map[Long, Seq[Long]]] object ServedTweetPreviewIdsFeature extends Feature[PipelineQuery, Seq[Long]] object SignupSourceFeature extends Feature[PipelineQuery, Option[SignupSource]] object SignupCountryFeature extends Feature[PipelineQuery, Option[String]] object ViewerAllowsAdsPersonalizationFeature extends Feature[PipelineQuery, Option[Boolean]] object ViewerAllowsForYouRecommendationsFeature extends Feature[PipelineQuery, Option[Boolean]] object ViewerAllowsDataSharingFeature extends Feature[PipelineQuery, Option[Boolean]] object TimelineServiceTweetsFeature extends Feature[PipelineQuery, Seq[Long]] object TLSOriginalTweetsWithAuthorFeature extends Feature[PipelineQuery, Seq[(Long, Option[Long])]] object TLSOriginalTweetsWithConfirmedAuthorFeature extends Feature[PipelineQuery, Seq[(Long, Long)]] object TweetAuthorFollowersFeature extends Feature[PipelineQuery, Map[Long, Option[Long]]] object TimestampFeature extends DataRecordFeature[PipelineQuery, Long] with LongDiscreteDataRecordCompatible { override val featureName: String = SharedFeatures.TIMESTAMP.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object TimestampGMTDowFeature extends DataRecordFeature[PipelineQuery, Long] with LongDiscreteDataRecordCompatible { override val featureName: String = RequestContextFeatures.TIMESTAMP_GMT_DOW.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object TimestampGMTHourFeature extends DataRecordFeature[PipelineQuery, Long] with LongDiscreteDataRecordCompatible { override val featureName: String = RequestContextFeatures.TIMESTAMP_GMT_HOUR.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object TweetMixerScoreFeature extends DataRecordOptionalFeature[TweetCandidate, Double] with DoubleDataRecordCompatible { override val featureName: String = TimelinesSharedFeatures.CANDIDATE_SOURCE_SCORE.getFeatureName override val personalDataTypes: Set[PersonalDataType] = Set(pd.PersonalDataType.EngagementScore) } object SourceSignalFeature extends Feature[TweetCandidate, Option[SourceSignal]] object DebugStringFeature extends Feature[TweetCandidate, Option[String]] object IsSelfThreadFeature extends DataRecordFeature[PipelineQuery, Boolean] with BoolDataRecordCompatible { override val featureName: String = conversation_features.ConversationFeatures.IS_SELF_THREAD_TWEET.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty } object TweetLanguageFromTweetypieFeature extends DataRecordOptionalFeature[TweetCandidate, String] with StringDataRecordCompatible { override val featureName: String = LanguageFeatures.TweetLanguageFromTweetypieFeature.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.InferredLanguage) } object TweetLanguageFromLanguageSignalFeature extends DataRecordOptionalFeature[PipelineQuery, String] with StringDataRecordCompatible { override val featureName: String = LanguageFeatures.TweetLanguageFromLanguageSignalFeature.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set( pd.PersonalDataType.InferredLanguage) } object UserActionsFeature extends Feature[PipelineQuery, Option[UserActionSequence]] object UserActionsByteArrayFeature extends Feature[PipelineQuery, Option[Array[Byte]]] object UserActionsSizeFeature extends Feature[PipelineQuery, Option[Int]] object UserActionsContainsExplicitSignalsFeature extends Feature[PipelineQuery, Boolean] object UserEngagedLanguagesFeature extends Feature[PipelineQuery, Set[String]] with SparseBinaryDataRecordCompatible { override val featureName: String = LanguageFeatures.EngagedLanguages.getFeatureName override val personalDataTypes: Set[pd.PersonalDataType] = Set( pd.PersonalDataType.InferredLanguage) } object UserFollowedTopicsCountFeature extends Feature[PipelineQuery, Option[Int]] object UserFollowingCountFeature extends Feature[PipelineQuery, Option[Int]] object UserFollowersCountFeature extends Feature[PipelineQuery, Option[Int]] object UserRecentEngagementTweetIdsFeature extends Feature[PipelineQuery, Seq[Long]] object UserLastExplicitSignalTimeFeature extends Feature[PipelineQuery, Option[Time]] object UserUnderstandableLanguagesFeature extends Feature[PipelineQuery, Seq[String]] object UserScreenNameFeature extends Feature[PipelineQuery, Option[String]] object UserStateFeature extends Feature[PipelineQuery, Option[um.UserState]] object UserTypeFeature extends Feature[PipelineQuery, Option[gt.UserType]] object ViewerSafetyLabels extends Feature[PipelineQuery, Option[Seq[String]]] object ViewerIsRateLimited extends Feature[PipelineQuery, Boolean] object ViewerHasJobRecommendationsEnabled extends Feature[PipelineQuery, Boolean] object ViewerHasPremiumTier extends Feature[PipelineQuery, Boolean] object ViewerHasRecruitingOrganizationRecommendationsEnabled extends Feature[PipelineQuery, Boolean] object WhoToFollowExcludedUserIdsFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { override def defaultValue = Seq.empty } object NSFWConsumerScoreFeature extends Feature[PipelineQuery, Double] object NSFWConsumerFollowerScoreFeature extends Feature[PipelineQuery, Double] object CurrentDisplayedGrokTopicFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Option[(Long, String)]] { override val defaultValue: Option[(Long, String)] = None } // Result Features object ServedSizeFeature extends Feature[PipelineQuery, Option[Int]] object UniqueAuthorCountFeature extends Feature[PipelineQuery, Option[Int]] object MaxSingleAuthorCountFeature extends Feature[PipelineQuery, Option[Int]] object UniqueCategoryCountFeature extends Feature[PipelineQuery, Option[Int]] object MaxSingleCategoryCountFeature extends Feature[PipelineQuery, Option[Int]] object ServedInConversationModuleFeature extends Feature[TweetCandidate, Boolean] object ConversationModule2DisplayedTweetsFeature extends Feature[TweetCandidate, Boolean] object ConversationModuleHasGapFeature extends Feature[TweetCandidate, Boolean] object SGSValidLikedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] object SGSValidFollowedByUserIdsFeature extends Feature[TweetCandidate, Seq[Long]] object ScreenNamesFeature extends Feature[TweetCandidate, Map[Long, String]] object RealNamesFeature extends Feature[TweetCandidate, Map[Long, String]] object TweetAgeFeature extends Feature[TweetCandidate, Option[Long]] /** * Features around the focal Tweet for Tweets which should be rendered in convo modules. * These are needed in order to render social context above the root tweet in a convo modules. * For example if we have a convo module A-B-C (A Tweets, B replies to A, C replies to B), the descendant features are * for the Tweet C. These features are None except for the root Tweet for Tweets which should render into * convo modules. */ object FocalTweetAuthorIdFeature extends Feature[TweetCandidate, Option[Long]] object FocalTweetInNetworkFeature extends Feature[TweetCandidate, Option[Boolean]] object FocalTweetRealNamesFeature extends Feature[TweetCandidate, Option[Map[Long, String]]] object FocalTweetScreenNamesFeature extends Feature[TweetCandidate, Option[Map[Long, String]]] object MediaUnderstandingAnnotationIdsFeature extends Feature[TweetCandidate, Seq[Long]] object AdTagUrlFeature extends Feature[TweetCandidate, Option[String]] } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/HomeLargeEmbeddingsFeatures.scala ================================================ package com.twitter.home_mixer.model import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.pipeline.PipelineQuery object HomeLargeEmbeddingsFeatures { object AuthorLargeEmbeddingsFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object AuthorLargeEmbeddingsKeyFeature extends Feature[TweetCandidate, Seq[Long]] object OriginalAuthorLargeEmbeddingsFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object OriginalAuthorLargeEmbeddingsKeyFeature extends Feature[TweetCandidate, Seq[Long]] object TweetLargeEmbeddingsFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object TweetLargeEmbeddingsKeyFeature extends Feature[TweetCandidate, Seq[Long]] object OriginalTweetLargeEmbeddingsFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object OriginalTweetLargeEmbeddingsKeyFeature extends Feature[TweetCandidate, Seq[Long]] object UserLargeEmbeddingsFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object UserLargeEmbeddingsKeyFeature extends Feature[PipelineQuery, Seq[Long]] } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/NaviClientConfig.scala ================================================ package com.twitter.home_mixer.model case class NaviClientConfig( clientName: String, customizedBatchSize: Option[Int], clusterStr: String) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/NavigationIncludeInstruction.scala ================================================ package com.twitter.home_mixer.model import com.twitter.home_mixer.model.request.DeviceContext.RequestContext import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.product_mixer.component_library.premarshaller.urt.builder.IncludeInstruction import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineEntry import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.configapi.FSParam /** * Include a navigation timeline instruction when we satisfy criteria */ case class NavigationIncludeInstruction( ptrEnableParam: FSParam[Boolean], coldStartEnableParam: FSParam[Boolean], warmStartEnableParam: FSParam[Boolean], navigateEnableParam: FSParam[Boolean], manualRefreshEnableParam: FSParam[Boolean]) extends IncludeInstruction[PipelineQuery with HasDeviceContext] { override def apply( query: PipelineQuery with HasDeviceContext, entries: Seq[TimelineEntry] ): Boolean = { val requestContext = query.deviceContext.flatMap(_.requestContextValue) val ptrEnabled = query.params(ptrEnableParam) && requestContext.contains(RequestContext.PullToRefresh) val coldStartEnabled = query.params(coldStartEnableParam) && requestContext.contains(RequestContext.Launch) val warmStartEnabled = query.params(warmStartEnableParam) && requestContext.contains(RequestContext.Foreground) val manualRefreshEnabled = query.params(manualRefreshEnableParam) && requestContext.contains( RequestContext.ManualRefresh) val navigateEnabled = query.params(navigateEnableParam) && requestContext.contains(RequestContext.Navigate) ptrEnabled || coldStartEnabled || warmStartEnabled || manualRefreshEnabled || navigateEnabled } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/PhoenixPredictedScoreFeature.scala ================================================ package com.twitter.home_mixer.model import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelWeights import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.UseProdInPhoenixParams import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam import com.x.user_action_sequence.ActionName import com.x.user_action_sequence.ActionName._ sealed trait PhoenixPredictedScoreFeature extends Feature[TweetCandidate, Option[Double]] { def featureName: String def modelWeightParam: FSBoundedParam[Double] def actions: Seq[ActionName] def isEligible(features: FeatureMap): Boolean = true def prodScoreFeature: PredictedScoreFeature def useProdFeatureParam: FSParam[Boolean] def extractScore(features: FeatureMap, query: PipelineQuery): Option[Double] = { if (query.params(useProdFeatureParam)) prodScoreFeature.extractScore(features, query) else features.getOrElse(this, None).orElse(prodScoreFeature.extractScore(features, query)) } } object PhoenixPredictedFavoriteScoreFeature extends PhoenixPredictedScoreFeature { override val featureName = "fav" override val modelWeightParam = ModelWeights.FavParam override val actions = Seq(SERVER_TWEET_FAV) override def prodScoreFeature = PredictedFavoriteScoreFeature override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdFavForPhoenixParam } object PhoenixPredictedReplyScoreFeature extends PhoenixPredictedScoreFeature { override val featureName = "reply" override val modelWeightParam = ModelWeights.ReplyParam override val actions = Seq(SERVER_TWEET_REPLY) override def prodScoreFeature = PredictedReplyScoreFeature override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdReplyForPhoenixParam } object PhoenixPredictedRetweetScoreFeature extends PhoenixPredictedScoreFeature { override val featureName = "retweet" override val modelWeightParam = ModelWeights.RetweetParam override val actions = Seq(SERVER_TWEET_QUOTE, SERVER_TWEET_RETWEET) override def prodScoreFeature = PredictedRetweetScoreFeature override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdRetweetForPhoenixParam } object PhoenixPredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature extends PhoenixPredictedScoreFeature { override val featureName = "click_engage" override val modelWeightParam = ModelWeights.GoodClickV1Param override val actions = Seq(CLIENT_TWEET_PHOTO_EXPAND) override def prodScoreFeature = PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdGoodClickV1ForPhoenixParam } object PhoenixPredictedGoodClickConvoDescUamGt2ScoreFeature extends PhoenixPredictedScoreFeature { override val featureName = "click_dwell" override val modelWeightParam = ModelWeights.GoodClickV2Param override val actions = Seq(CLIENT_TWEET_CLICK) override def prodScoreFeature = PredictedGoodClickConvoDescUamGt2ScoreFeature override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdGoodClickV2ForPhoenixParam } object PhoenixPredictedGoodProfileClickScoreFeature extends PhoenixPredictedScoreFeature { override val featureName = "good_profile_click" override val modelWeightParam = ModelWeights.GoodProfileClickParam override val actions = Seq(CLIENT_TWEET_CLICK_PROFILE) override def prodScoreFeature = PredictedGoodProfileClickScoreFeature override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdProfileClickForPhoenixParam } object PhoenixPredictedVideoQualityViewScoreFeature extends PhoenixPredictedScoreFeature { override val featureName = "vqv" override val modelWeightParam = ModelWeights.VideoQualityViewParam override val actions = Seq(CLIENT_TWEET_VIDEO_QUALITY_VIEW) override def isEligible(features: FeatureMap): Boolean = { val isVideoDurationGte10Seconds = (features.getOrElse(VideoDurationMsFeature, None).getOrElse(0) / 1000.0) >= 10 val hasVideoFeature = features.getOrElse(HasVideoFeature, false) hasVideoFeature && isVideoDurationGte10Seconds } override def prodScoreFeature = PredictedVideoQualityViewScoreFeature override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdVQVForPhoenixParam } object PhoenixPredictedShareScoreFeature extends PhoenixPredictedScoreFeature { override val featureName = "share" override val modelWeightParam = ModelWeights.ShareParam override val actions = Seq( CLIENT_TWEET_SHARE_VIA_COPY_LINK, CLIENT_TWEET_CLICK_SEND_VIA_DIRECT_MESSAGE, CLIENT_TWEET_SHARE ) override def prodScoreFeature = PredictedShareScoreFeature override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdShareForPhoenixParam } object PhoenixPredictedDwellScoreFeature extends PhoenixPredictedScoreFeature { override val featureName = "dwell" override val modelWeightParam = ModelWeights.DwellParam override val actions = Seq(CLIENT_TWEET_RECAP_DWELLED) override def isEligible(features: FeatureMap): Boolean = { val isVideoDurationGte10Seconds = (features.getOrElse(VideoDurationMsFeature, None).getOrElse(0) / 1000.0) >= 10 val hasVideoFeature = features.getOrElse(HasVideoFeature, false) !(hasVideoFeature && isVideoDurationGte10Seconds) } override def prodScoreFeature = PredictedDwellScoreFeature override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdDwellForPhoenixParam } object PhoenixPredictedOpenLinkScoreFeature extends PhoenixPredictedScoreFeature { override val featureName = "open_link" override val modelWeightParam = ModelWeights.OpenLinkParam override val actions = Seq(CLIENT_TWEET_OPEN_LINK) // Placeholder prod score feature. This should not be used. override def prodScoreFeature = PredictedShareScoreFeature override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdOpenLinkForPhoenixParam } object PhoenixPredictedScreenshotScoreFeature extends PhoenixPredictedScoreFeature { override val featureName = "screenshot" override val modelWeightParam = ModelWeights.OpenLinkParam override val actions = Seq(CLIENT_TWEET_TAKE_SCREENSHOT) // Placeholder prod score feature. This should not be used. override def prodScoreFeature = PredictedShareScoreFeature override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdScreenshotForPhoenixParam } object PhoenixPredictedBookmarkScoreFeature extends PhoenixPredictedScoreFeature { override val featureName = "bookmark" override val modelWeightParam = ModelWeights.BookmarkParam override val actions = Seq(CLIENT_TWEET_BOOKMARK) // Placeholder prod score feature. This should not be used. override def prodScoreFeature = PredictedBookmarkScoreFeature override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdBookmarkForPhoenixParam } // Negative Engagements object PhoenixPredictedNegativeFeedbackV2ScoreFeature extends PhoenixPredictedScoreFeature { override val featureName = "negative_feedback_v2" override val modelWeightParam = ModelWeights.NegativeFeedbackV2Param override val actions = Seq( CLIENT_TWEET_NOT_INTERESTED_IN, CLIENT_TWEET_BLOCK_AUTHOR, CLIENT_TWEET_MUTE_AUTHOR, CLIENT_TWEET_REPORT, ) override def prodScoreFeature = PredictedNegativeFeedbackV2ScoreFeature override def useProdFeatureParam = UseProdInPhoenixParams.EnableProdNegForPhoenixParam } object PhoenixPredictedScoreFeature { val PhoenixPredictedScoreFeatures: Seq[PhoenixPredictedScoreFeature] = Seq( PhoenixPredictedFavoriteScoreFeature, PhoenixPredictedReplyScoreFeature, PhoenixPredictedRetweetScoreFeature, PhoenixPredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, PhoenixPredictedGoodClickConvoDescUamGt2ScoreFeature, PhoenixPredictedGoodProfileClickScoreFeature, PhoenixPredictedVideoQualityViewScoreFeature, PhoenixPredictedShareScoreFeature, PhoenixPredictedDwellScoreFeature, PhoenixPredictedOpenLinkScoreFeature, PhoenixPredictedScreenshotScoreFeature, PhoenixPredictedBookmarkScoreFeature, // Negative Engagements PhoenixPredictedNegativeFeedbackV2ScoreFeature, ) val PhoenixPredictedScoreFeatureSet: Set[PhoenixPredictedScoreFeature] = PhoenixPredictedScoreFeatures.toSet } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/PredictedScoreFeature.scala ================================================ package com.twitter.home_mixer.model import com.twitter.dal.personal_data.thriftjava.PersonalDataType import com.twitter.dal.personal_data.{thriftjava => pd} import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableImmersiveVQV import com.twitter.home_mixer.param.HomeGlobalParams.EnableTenSecondsLogicForVQV import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ScoreThresholdForVQVParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.EnableBinarySchemeForVQVParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.BinarySchemeConstantForVQVParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ScoreThresholdForDwellParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.EnableDwellOrVQVParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.EnableBinarySchemeForDwellParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelBiases import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelDebiases import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelWeights import com.twitter.ml.api.DataType import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.prediction.features.recap.RecapFeatures import com.twitter.ml.api.thriftscala.GeneralTensor import com.twitter.product_mixer.core.feature.datarecord.GeneralTensorDataRecordCompatible sealed trait PredictedScoreFeature extends DataRecordOptionalFeature[TweetCandidate, Double] with DoubleDataRecordCompatible { override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty // Note: Determine whether the score prediction should contribute to the weighted score // Currently, it is being used for PredictedVideoQualityViewScoreFeature def isEligible( features: FeatureMap, query: PipelineQuery ): Boolean = true def statName: String def modelWeightParam: FSBoundedParam[Double] def modelBiasParam: Option[FSBoundedParam[Double]] = None def modelDebiasParam: Option[FSBoundedParam[Double]] = None def extractScore(features: FeatureMap, query: PipelineQuery): Option[Double] = features.getOrElse(this, None) def weightQueryFeatureName: String lazy val weightQueryFeature: Feature[PipelineQuery, Option[Double]] = PredictedScoreFeature.getDataRecordFeatureFromName(weightQueryFeatureName) def biasQueryFeatureName: Option[String] = None lazy val biasQueryFeature: Option[Feature[PipelineQuery, Option[Double]]] = biasQueryFeatureName.map(PredictedScoreFeature.getDataRecordFeatureFromName) def debiasQueryFeatureName: Option[String] = None lazy val debiasQueryFeature: Option[Feature[PipelineQuery, Option[GeneralTensor]]] = debiasQueryFeatureName.map(PredictedScoreFeature.getGeneralTensorFeatureFromName) } object PredictedFavoriteScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_FAVORITED.getFeatureName override val statName = "fav" override val modelWeightParam = ModelWeights.FavParam override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_FAVORITED.getFeatureName override val modelDebiasParam = Some(ModelDebiases.FavParam) override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_FAVORITED.getFeatureName) } object PredictedReplyScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_REPLIED.getFeatureName override val statName = "reply" override val modelWeightParam = ModelWeights.ReplyParam override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_REPLIED.getFeatureName override val modelDebiasParam = Some(ModelDebiases.ReplyParam) override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_REPLIED.getFeatureName) } object PredictedRetweetScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_RETWEETED.getFeatureName override val statName = "retweet" override val modelWeightParam = ModelWeights.RetweetParam override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_RETWEETED.getFeatureName override val modelDebiasParam = Some(ModelDebiases.RetweetParam) override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_RETWEETED.getFeatureName) } object PredictedReplyEngagedByAuthorScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_REPLIED_REPLY_ENGAGED_BY_AUTHOR.getFeatureName override val statName = "reply_engaged_by_author" override val modelWeightParam = ModelWeights.ReplyEngagedByAuthorParam override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_REPLIED_REPLY_ENGAGED_BY_AUTHOR.getFeatureName override val modelDebiasParam = Some(ModelDebiases.ReplyEngagedByAuthorParam) override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_REPLIED_REPLY_ENGAGED_BY_AUTHOR.getFeatureName) } object PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_GOOD_CLICKED_V1.getFeatureName override val statName = "click_engaged" // click_convo_desc_favorited_or_replied override val modelWeightParam = ModelWeights.GoodClickV1Param override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_GOOD_CLICKED_V1.getFeatureName override val modelDebiasParam = Some(ModelDebiases.GoodClickV1Param) override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_GOOD_CLICKED_V1.getFeatureName) } object PredictedGoodClickConvoDescUamGt2ScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_GOOD_CLICKED_V2.getFeatureName override val statName = "click_dwell" // good_click_convo_desc_uam_gt_2 override val modelWeightParam = ModelWeights.GoodClickV2Param override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_GOOD_CLICKED_V2.getFeatureName override val modelDebiasParam = Some(ModelDebiases.GoodClickV2Param) override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_GOOD_CLICKED_V2.getFeatureName) } object PredictedGoodProfileClickScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_PROFILE_CLICKED_AND_PROFILE_ENGAGED.getFeatureName override val statName = "good_profile_click" override val modelWeightParam = ModelWeights.GoodProfileClickParam override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_PROFILE_CLICKED_AND_PROFILE_ENGAGED.getFeatureName override val modelDebiasParam = Some(ModelDebiases.GoodProfileClickParam) override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_PROFILE_CLICKED_AND_PROFILE_ENGAGED.getFeatureName) } object PredictedVideoQualityViewImmersiveScoreFeature extends PredictedScoreFeature { override def isEligible( features: FeatureMap, query: PipelineQuery ): Boolean = { query.params(EnableImmersiveVQV) } override val featureName: String = RecapFeatures.PREDICTED_IS_VIDEO_QUALITY_VIEWED_IMMERSIVE.getFeatureName override val statName = "vqv_immersive" // video_quality_viewed_immersive override val modelWeightParam = ModelWeights.VideoQualityViewImmersiveParam override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_VIDEO_QUALITY_VIEWED_IMMERSIVE.getFeatureName override val modelBiasParam = Some(ModelBiases.VideoQualityViewImmersiveParam) override val biasQueryFeatureName = Some( RecapFeatures.BIAS_IS_VIDEO_QUALITY_VIEWED_IMMERSIVE.getFeatureName) override val modelDebiasParam = Some(ModelDebiases.VideoQualityViewImmersiveParam) override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_VIDEO_QUALITY_VIEWED_IMMERSIVE.getFeatureName) } object PredictedVideoQualityViewScoreFeature extends PredictedScoreFeature { override def isEligible( features: FeatureMap, query: PipelineQuery ): Boolean = { val isTenSecondsLogicEnabled = query.params(EnableTenSecondsLogicForVQV) val isVideoDurationGte10Seconds = (features.getOrElse(VideoDurationMsFeature, None).getOrElse(0) / 1000.0) >= 10 val hasVideoFeature = features.getOrElse(HasVideoFeature, false) hasVideoFeature && (!isTenSecondsLogicEnabled || isVideoDurationGte10Seconds) } override val featureName: String = RecapFeatures.PREDICTED_IS_VIDEO_QUALITY_VIEWED.getFeatureName override val statName = "vqv" // video_quality_viewed override val modelWeightParam = ModelWeights.VideoQualityViewParam override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_VIDEO_QUALITY_VIEWED.getFeatureName override val modelBiasParam = Some(ModelBiases.VideoQualityViewParam) override val biasQueryFeatureName = Some( RecapFeatures.BIAS_IS_VIDEO_QUALITY_VIEWED.getFeatureName) override val modelDebiasParam = Some(ModelDebiases.VideoQualityViewParam) override val debiasQueryFeatureName = Some( RecapFeatures.DEBIAS_IS_VIDEO_QUALITY_VIEWED.getFeatureName) override def extractScore(features: FeatureMap, query: PipelineQuery): Some[Double] = { // For VQV, if the score is below a threshold, we return 0 val vqvScore = features.getOrElse(this, None).getOrElse(0.0) if (vqvScore < query.params(ScoreThresholdForVQVParam)) { // the default threshold is 0.0, vqvScore should be always non-negative Some(0.0) } else if (query.params(EnableBinarySchemeForVQVParam)) { // If the binary scheme is enabled, we return a constant Some(query.params(BinarySchemeConstantForVQVParam)) } else { Some(vqvScore) } } } object PredictedBookmarkScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_BOOKMARKED.getFeatureName override val statName = "bookmark" override val modelWeightParam = ModelWeights.BookmarkParam override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_BOOKMARKED.getFeatureName override val modelDebiasParam = Some(ModelDebiases.BookmarkParam) override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_BOOKMARKED.getFeatureName) } object PredictedShareScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_SHARED.getFeatureName override val statName = "share" override val modelWeightParam = ModelWeights.ShareParam override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_SHARED.getFeatureName override val modelDebiasParam = Some(ModelDebiases.ShareParam) override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_SHARED.getFeatureName) } object PredictedDwellScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_DWELLED.getFeatureName override val statName = "dwell" override val modelWeightParam = ModelWeights.DwellParam override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_DWELLED.getFeatureName override val modelDebiasParam = Some(ModelDebiases.DwellParam) override val debiasQueryFeatureName = Some(RecapFeatures.DEBIAS_IS_DWELLED.getFeatureName) override def isEligible( features: FeatureMap, query: PipelineQuery ): Boolean = { val isTenSecondsLogicEnabled = query.params(EnableTenSecondsLogicForVQV) val isVideoDurationGte10Seconds = (features.getOrElse(VideoDurationMsFeature, None).getOrElse(0) / 1000.0) >= 10 val hasVideoFeature = features.getOrElse(HasVideoFeature, false) val isEligibleForVqv = hasVideoFeature && (!isTenSecondsLogicEnabled || isVideoDurationGte10Seconds) !(query.params(EnableDwellOrVQVParam) && isEligibleForVqv) } override def extractScore(features: FeatureMap, query: PipelineQuery): Some[Double] = { // For Dwell, if the score is below a threshold, we return 0 val dwellScore = features.getOrElse(this, None).getOrElse(0.0) if (dwellScore < query.params(ScoreThresholdForDwellParam)) { // the default threshold is 0.0, dwellScore should be always non-negative Some(0.0) } else if (query.params(EnableBinarySchemeForDwellParam)) { // If the binary scheme is enabled, we return a constant Some(query.params(ScoreThresholdForDwellParam)) } else { Some(dwellScore) } } } object PredictedVideoWatchTimeScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_VIDEO_WATCH_TIME_MS.getFeatureName override val statName = "video_watch_time_ms" override val modelWeightParam = ModelWeights.VideoWatchTimeMsParam override val weightQueryFeatureName = RecapFeatures.WEIGHT_VIDEO_WATCH_TIME_MS.getFeatureName override val modelDebiasParam = Some(ModelDebiases.VideoWatchTimeMsParam) override val debiasQueryFeatureName = Some( RecapFeatures.DEBIAS_VIDEO_WATCH_TIME_MS.getFeatureName) } // Negative Engagements object PredictedNegativeFeedbackV2ScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_NEGATIVE_FEEDBACK_V2.getFeatureName override val statName = "negative_feedback_v2" override val modelWeightParam = ModelWeights.NegativeFeedbackV2Param override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_NEGATIVE_FEEDBACK_V2.getFeatureName override val modelDebiasParam = Some(ModelDebiases.NegativeFeedbackV2Param) override val debiasQueryFeatureName = Some( RecapFeatures.DEBIAS_IS_NEGATIVE_FEEDBACK_V2.getFeatureName) } object PredictedVideoQualityWatchScoreFeature extends PredictedScoreFeature { override def isEligible( features: FeatureMap, query: PipelineQuery ) = { features.getOrElse(HasVideoFeature, false) && (features .getOrElse(VideoDurationMsFeature, None).getOrElse(0) / 1000.0) >= 10 } override val featureName: String = RecapFeatures.PREDICTED_IS_VIDEO_QUALITY_WATCH.getFeatureName override val statName = "video_quality_watched" override val modelWeightParam = ModelWeights.VideoQualityWatchParam override val weightQueryFeatureName = RecapFeatures.WEIGHT_IS_VIDEO_QUALITY_WATCHED.getFeatureName override val modelBiasParam = Some(ModelBiases.VideoQualityWatchParam) override val biasQueryFeatureName = Some( RecapFeatures.BIAS_IS_VIDEO_QUALITY_WATCHED.getFeatureName) override val modelDebiasParam = Some(ModelDebiases.VideoQualityWatchParam) override val debiasQueryFeatureName = Some( RecapFeatures.DEBIAS_IS_VIDEO_QUALITY_WATCHED.getFeatureName) } object PredictedScoreFeature { val PredictedScoreFeatures: Seq[PredictedScoreFeature] = Seq( PredictedFavoriteScoreFeature, PredictedReplyScoreFeature, PredictedRetweetScoreFeature, PredictedReplyEngagedByAuthorScoreFeature, PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, PredictedGoodClickConvoDescUamGt2ScoreFeature, PredictedGoodProfileClickScoreFeature, PredictedVideoQualityViewScoreFeature, PredictedVideoQualityViewImmersiveScoreFeature, PredictedBookmarkScoreFeature, PredictedShareScoreFeature, PredictedDwellScoreFeature, PredictedVideoQualityWatchScoreFeature, PredictedVideoWatchTimeScoreFeature, // Negative Engagements PredictedNegativeFeedbackV2ScoreFeature, ) val PredictedScoreFeatureSet: Set[PredictedScoreFeature] = PredictedScoreFeatures.toSet def getGeneralTensorFeatureFromName( name: String ): Feature[PipelineQuery, Option[GeneralTensor]] = { new DataRecordOptionalFeature[PipelineQuery, GeneralTensor] with GeneralTensorDataRecordCompatible { override val featureName: String = name override val personalDataTypes: Set[PersonalDataType] = Set.empty override def toString: String = name override def dataType: DataType = DataType.DOUBLE } } def getDataRecordFeatureFromName( name: String ): Feature[PipelineQuery, Option[Double]] = { new DataRecordOptionalFeature[PipelineQuery, Double] with DoubleDataRecordCompatible { override val featureName: String = name override val personalDataTypes: Set[PersonalDataType] = Set.empty override def toString: String = name } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = ["user-signal-service/thrift/src/main/thrift:thrift-scala"], exports = [], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source/SourceSignal.scala ================================================ package com.twitter.home_mixer.model.candidate_source import com.twitter.usersignalservice.{thriftscala => se} case class SourceSignal( id: Long, signalType: Option[String], signalEntity: Option[se.SignalEntity], authorId: Option[Long]) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/signup", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", "timelineservice/common:model", ], exports = [ "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/DeviceContext.scala ================================================ package com.twitter.home_mixer.model.request import com.twitter.product_mixer.core.model.marshalling.request.ClientContext import com.twitter.{timelineservice => tls} case class DeviceContext( isPolling: Option[Boolean], requestContext: Option[String], latestControlAvailable: Option[Boolean], autoplayEnabled: Option[Boolean]) { lazy val requestContextValue: Option[DeviceContext.RequestContext.Value] = requestContext.flatMap { value => val normalizedValue = value.trim.toLowerCase() DeviceContext.RequestContext.values.find(_.toString == normalizedValue) } def toTimelineServiceDeviceContext(clientContext: ClientContext): tls.DeviceContext = tls.DeviceContext( countryCode = clientContext.countryCode, languageCode = clientContext.languageCode, clientAppId = clientContext.appId, ipAddress = clientContext.ipAddress, guestId = clientContext.guestId, sessionId = None, timezone = None, userAgent = clientContext.userAgent, deviceId = clientContext.deviceId, isPolling = isPolling, requestProvenance = requestContext, referrer = None, tfeAuthHeader = None, mobileDeviceId = clientContext.mobileDeviceId, isSessionStart = None, displaySize = None, isURTRequest = Some(true), latestControlAvailable = latestControlAvailable, guestIdMarketing = clientContext.guestIdMarketing, isInternalOrTwoffice = clientContext.isTwoffice, browserNotificationPermission = None, guestIdAds = clientContext.guestIdAds, ) } object DeviceContext { val Empty: DeviceContext = DeviceContext( isPolling = None, requestContext = None, latestControlAvailable = None, autoplayEnabled = None ) /** * Constants which reflect valid client request provenances (why a request was initiated, encoded * by the "request_context" HTTP parameter). */ object RequestContext extends Enumeration { val Auto = Value("auto") val Foreground = Value("foreground") val Gap = Value("gap") val Launch = Value("launch") val ManualRefresh = Value("manual_refresh") val Navigate = Value("navigate") val Polling = Value("polling") val PullToRefresh = Value("ptr") val Signup = Value("signup") val TweetSelfThread = Value("tweet_self_thread") val BackgroundFetch = Value("background_fetch") } } trait HasDeviceContext { def deviceContext: Option[DeviceContext] } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasListId.scala ================================================ package com.twitter.home_mixer.model.request /** * [[HasListId]] enables shared components to access the list id shared by all list timeline products. */ trait HasListId { def listId: Long } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HasSeenTweetIds.scala ================================================ package com.twitter.home_mixer.model.request /** * [[HasSeenTweetIds]] enables shared components to access the list of impressed tweet IDs * sent by clients across different Home Mixer query types (e.g. FollowingQuery, ForYouQuery) */ trait HasSeenTweetIds { def seenTweetIds: Option[Seq[Long]] } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerDebugOptions.scala ================================================ package com.twitter.home_mixer.model.request import com.twitter.product_mixer.core.model.marshalling.request.DebugOptions import com.twitter.util.Time case class HomeMixerDebugOptions( override val requestTimeOverride: Option[Time], override val showIntermediateLogs: Option[Boolean]) extends DebugOptions ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProduct.scala ================================================ package com.twitter.home_mixer.model.request import com.twitter.product_mixer.core.model.common.identifier.ProductIdentifier import com.twitter.product_mixer.core.model.marshalling.request.Product /** * Identifier names on products can be used to create Feature Switch rules by product, * which useful if bucketing occurs in a component shared by multiple products. * @see [[Product.identifier]] */ case object FollowingProduct extends Product { override val identifier: ProductIdentifier = ProductIdentifier("Following") override val stringCenterProject: Option[String] = Some("timelinemixer") } case object ForYouProduct extends Product { override val identifier: ProductIdentifier = ProductIdentifier("ForYou") override val stringCenterProject: Option[String] = Some("timelinemixer") } case object ScoredTweetsProduct extends Product { override val identifier: ProductIdentifier = ProductIdentifier("ScoredTweets") override val stringCenterProject: Option[String] = Some("timelinemixer") } case object ScoredVideoTweetsProduct extends Product { override val identifier: ProductIdentifier = ProductIdentifier("ScoredVideoTweets") override val stringCenterProject: Option[String] = Some("timelinemixer") } case object SubscribedProduct extends Product { override val identifier: ProductIdentifier = ProductIdentifier("Subscribed") override val stringCenterProject: Option[String] = Some("timelinemixer") } case object HeavyRankerScoresProduct extends Product { override val identifier: ProductIdentifier = ProductIdentifier("HeavyRankerScores") override val stringCenterProject: Option[String] = Some("timelinemixer") } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerProductContext.scala ================================================ package com.twitter.home_mixer.model.request import com.twitter.dspbidder.commons.thriftscala.DspClientContext import com.twitter.home_mixer.model.signup.SignupSource import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.core.model.marshalling.request.ProductContext case class FollowingProductContext( deviceContext: Option[DeviceContext], seenTweetIds: Option[Seq[Long]], dspClientContext: Option[DspClientContext]) extends ProductContext case class ForYouProductContext( deviceContext: Option[DeviceContext], seenTweetIds: Option[Seq[Long]], dspClientContext: Option[DspClientContext]) extends ProductContext case class ScoredTweetsProductContext( deviceContext: Option[DeviceContext], seenTweetIds: Option[Seq[Long]], servedTweetIds: Option[Seq[Long]], backfillTweetIds: Option[Seq[Long]], signupCountryCode: Option[String], allowForYouRecommendations: Option[Boolean], signupSource: Option[SignupSource], followerCount: Option[Int], servedAuthorIds: Option[Map[Long, Seq[Long]]] = None) extends ProductContext case class ScoredVideoTweetsProductContext( deviceContext: Option[DeviceContext], seenTweetIds: Option[Seq[Long]], videoType: Option[t.VideoType], pinnedRelatedTweetIds: Option[Seq[Long]], scorePinnedTweetsOnly: Option[Boolean], immersiveClientMetadata: Option[t.ImmersiveClientMetadata]) extends ProductContext case class SubscribedProductContext( deviceContext: Option[DeviceContext], seenTweetIds: Option[Seq[Long]]) extends ProductContext case class HeavyRankerScoresProductContext( deviceContext: Option[DeviceContext], tweetIds: Option[Seq[Long]]) extends ProductContext ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request/HomeMixerRequest.scala ================================================ package com.twitter.home_mixer.model.request import com.twitter.product_mixer.core.model.marshalling.request.ClientContext import com.twitter.product_mixer.core.model.marshalling.request.DebugParams import com.twitter.product_mixer.core.model.marshalling.request.Product import com.twitter.product_mixer.core.model.marshalling.request.ProductContext import com.twitter.product_mixer.core.model.marshalling.request.Request case class HomeMixerRequest( override val clientContext: ClientContext, override val product: Product, // Product-specific parameters should be placed in the Product Context override val productContext: Option[ProductContext], override val serializedRequestCursor: Option[String], override val maxResults: Option[Int], override val debugParams: Option[DebugParams], // Parameters that apply to all products can be promoted to the request-level homeRequestParam: Boolean) extends Request ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/signup/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/model/signup/SignupSource.scala ================================================ package com.twitter.home_mixer.model.signup sealed trait SignupSource case object Onboard extends SignupSource case object MarchMadness extends SignupSource ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/AdvertiserBrandSafetySettingsStoreModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.adserver.{thriftscala => ads} import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.storage.client.manhattan.kv.Guarantee import com.twitter.storehaus.ReadableStore import com.twitter.storehaus_internal.manhattan.ManhattanCluster import com.twitter.storehaus_internal.manhattan.ManhattanClusters import com.twitter.timelines.clients.ads.AdvertiserBrandSafetySettingsStore import com.twitter.timelines.clients.manhattan.mhv3.ManhattanClientBuilder import com.twitter.timelines.clients.manhattan.mhv3.ManhattanClientConfigWithDataset import com.twitter.util.Duration import javax.inject.Singleton object AdvertiserBrandSafetySettingsStoreModule extends TwitterModule { @Provides @Singleton def providesAdvertiserBrandSafetySettingsStore( injectedServiceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): ReadableStore[Long, ads.AdvertiserBrandSafetySettings] = { val advertiserBrandSafetySettingsManhattanClientConfig = new ManhattanClientConfigWithDataset { override val cluster: ManhattanCluster = ManhattanClusters.apollo override val appId: String = "brand_safety_apollo" override val dataset = "advertiser_brand_safety_settings" override val statsScope: String = "AdvertiserBrandSafetySettingsManhattanClient" override val defaultGuarantee = Guarantee.Weak override val defaultMaxTimeout: Duration = 100.milliseconds override val maxRetryCount: Int = 1 override val isReadOnly: Boolean = true override val serviceIdentifier: ServiceIdentifier = injectedServiceIdentifier } val advertiserBrandSafetySettingsManhattanEndpoint = ManhattanClientBuilder .buildManhattanEndpoint(advertiserBrandSafetySettingsManhattanClientConfig, statsReceiver) val advertiserBrandSafetySettingsStore: ReadableStore[Long, ads.AdvertiserBrandSafetySettings] = AdvertiserBrandSafetySettingsStore .cached( advertiserBrandSafetySettingsManhattanEndpoint, advertiserBrandSafetySettingsManhattanClientConfig.dataset, ttl = 60.minutes, maxKeys = 100000, windowSize = 10L )(statsReceiver) advertiserBrandSafetySettingsStore } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/bijection:scrooge", "3rdparty/jvm/com/twitter/storehaus:core", "3rdparty/jvm/io/grpc:grpc-netty", "cproxy/thrift/src/main/thrift:thrift-scala", "deferredrpc/client/src/main/scala", "deferredrpc/client/src/main/thrift:thrift-scala", "escherbird/src/scala/com/twitter/escherbird/util/uttclient", "escherbird/src/thrift/com/twitter/escherbird/utt:strato-columns-scala", "eventbus/client/src/main/scala/com/twitter/eventbus/client", "events-recos/events-recos-service/src/main/thrift:events-recos-thrift-scala", "finagle-internal/finagle-grpc/src/main/scala", "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/interests:package", "graph-feature-service/src/main/thrift/com/twitter/graph_feature_service:graph_feature_service_thrift-scala", "home-mixer-features/thrift/src/main/thrift:thrift-java", "home-mixer-features/thrift/src/main/thrift:thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/store", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", "home-mixer/thrift/src/main/thrift:thrift-scala", "interests-service/thrift/src/main/thrift:thrift-scala", "kafka/finagle-kafka/finatra-kafka/src/main/scala", "limiter/thrift-only/src/main/thrift:thrift-scala", "media-understanding/video-summary/thrift/src/main/thrift:thrift-scala", "people-discovery/api/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/manhattan_client", "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/memcached_client", "product-mixer/shared-library/src/main/scala/com/twitter/product_mixer/shared_library/thrift_client", "servo/manhattan", "src/java/com/twitter/logpipeline/client:event-publisher-client-lib", "src/java/com/twitter/logpipeline/client/common:event-publisher-core", "src/java/com/twitter/logpipeline/client/serializers:event-publisher-serializers", "src/scala/com/twitter/ml/featurestore/lib", "src/scala/com/twitter/scalding_internal/multiformat/format", "src/scala/com/twitter/simclusters_v2/common", "src/scala/com/twitter/storehaus_internal", "src/scala/com/twitter/storehaus_internal/offline", "src/scala/com/twitter/summingbird_internal/bijection:bijection-implicits", "src/scala/com/twitter/summingbird_internal/runner/store_config", "src/scala/com/twitter/timelines/util", "src/scala/com/twitter/wtf/entity_real_graph/summingbird/client", "src/scala/com/twitter/wtf/entity_real_graph/summingbird/common/config", "src/thrift/com/twitter/ads/adserver:adserver_rpc-scala", "src/thrift/com/twitter/clientapp/gen:clientapp-scala", "src/thrift/com/twitter/manhattan:v1-scala", "src/thrift/com/twitter/manhattan:v2-scala", "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", "src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/service/metastore/gen:thrift-scala", "src/thrift/com/twitter/timelines/impression_store:thrift-scala", "src/thrift/com/twitter/timelines/realtime_aggregates:thrift-scala", "src/thrift/com/twitter/timelines/served_candidates_logging:served_candidates_logging-scala", "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", "src/thrift/com/twitter/user_session_store:thrift-java", "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", "stitch/stitch-tweetypie", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/rankedtweetcaching", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", "timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/scored_candidate_features_cache", "timelines/src/main/scala/com/twitter/timelines/clients/ads", "timelines/src/main/scala/com/twitter/timelines/clients/manhattan", "timelines/src/main/scala/com/twitter/timelines/clients/memcache_common", "timelines/src/main/scala/com/twitter/timelines/clients/predictionservice", "timelines/src/main/scala/com/twitter/timelines/clients/strato/topics", "timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly", "timelines/src/main/scala/com/twitter/timelines/config", "timelines/src/main/scala/com/twitter/timelines/util/stats", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", "topic-social-proof/server/src/main/scala/com/twitter/tsp/stores", "topiclisting/common/src/main/scala/com/twitter/topiclisting/clients/utt", "topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting", "topiclisting/topiclisting-utt/src/main/scala/com/twitter/topiclisting/utt", "tweetconvosvc/client/src/main/scala/com/twitter/tweetconvosvc/client/builder", "user_history_transformer/protobuf/src/main/protobuf/com/x/user_action_sequence", "user_history_transformer/service/src/main/java/com/x/user_action_sequence", ], exports = [ "timelines/src/main/scala/com/twitter/timelines/clients/predictionservice", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/BlenderClientModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.thrift.ClientId import com.twitter.inject.TwitterModule import com.twitter.product_mixer.shared_library.thrift_client.FinagleThriftClientBuilder import com.twitter.product_mixer.shared_library.thrift_client.NonIdempotent import com.twitter.search.blender.thriftscala.BlenderService import javax.inject.Singleton object BlenderClientModule extends TwitterModule { @Singleton @Provides def providesBlenderClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): BlenderService.MethodPerEndpoint = { val clientId = serviceIdentifier.environment.toLowerCase match { case "prod" => ClientId("") case _ => ClientId("") } FinagleThriftClientBuilder.buildFinagleMethodPerEndpoint[ BlenderService.ServicePerEndpoint, BlenderService.MethodPerEndpoint ]( serviceIdentifier = serviceIdentifier, clientId = clientId, dest = "/s/blender-universal/blender", label = "blender", statsReceiver = statsReceiver, idempotency = NonIdempotent, timeoutPerRequest = 1000.milliseconds, timeoutTotal = 1000.milliseconds, ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ClientSentImpressionsPublisherModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.eventbus.client.EventBusPublisher import com.twitter.eventbus.client.EventBusPublisherBuilder import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.timelines.config.ConfigUtils import com.twitter.timelines.config.Env import com.twitter.timelines.impressionstore.thriftscala.PublishedImpressionList import javax.inject.Singleton object ClientSentImpressionsPublisherModule extends TwitterModule with ConfigUtils { private val serviceName = "home-mixer" @Singleton @Provides def providesClientSentImpressionsPublisher( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): EventBusPublisher[PublishedImpressionList] = { val env = serviceIdentifier.environment.toLowerCase match { case "prod" => Env.prod case "staging" => Env.staging case "local" => Env.local case _ => Env.devel } val streamName = env match { case Env.prod => "timelinemixer_client_sent_impressions_prod" case _ => "timelinemixer_client_sent_impressions_devel" } EventBusPublisherBuilder() .clientId(clientIdWithScopeOpt(serviceName, env)) .serviceIdentifier(serviceIdentifier) .streamName(streamName) .statsReceiver(statsReceiver.scope("eventbus")) .thriftStruct(PublishedImpressionList) .tcpConnectTimeout(20.milliseconds) .connectTimeout(100.milliseconds) .requestTimeout(1.second) .publishTimeout(1.second) .build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ClusterDetailsModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.bijection.Bufferable import com.twitter.bijection.Injection import com.twitter.bijection.scrooge.CompactScalaCodec import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.thriftscala.ClusterDetails import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams import com.twitter.storehaus.ReadableStore import com.twitter.storehaus_internal.manhattan.Athena import com.twitter.storehaus_internal.manhattan.ManhattanRO import com.twitter.storehaus_internal.manhattan.ManhattanROConfig import com.twitter.storehaus_internal.util.ApplicationID import com.twitter.storehaus_internal.util.DatasetName import com.twitter.storehaus_internal.util.HDFSPath import javax.inject.Singleton object ClusterDetailsModule extends TwitterModule { @Provides @Singleton def providesClusterDetailsStore( serviceIdentifier: ServiceIdentifier ): ReadableStore[String, ClusterDetails] = { implicit val keyInjection: Injection[(String, Int), Array[Byte]] = Bufferable.injectionOf[(String, Int)] implicit val valueInjection: Injection[ClusterDetails, Array[Byte]] = CompactScalaCodec(ClusterDetails) val modelName = "20M_145K_2020" ManhattanRO .getReadableStoreWithMtls[(String, Int), ClusterDetails]( ManhattanROConfig( HDFSPath(""), ApplicationID("simclusters_v2"), DatasetName("simclusters_v2_cluster_details_20m_145k_2020"), Athena ), ManhattanKVClientMtlsParams(serviceIdentifier) ).composeKeyMapping(clusterIdString => (modelName, clusterIdString.toInt)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ConversationServiceModule.scala ================================================ package com.twitter.home_mixer.module import com.twitter.conversions.DurationOps._ import com.twitter.finagle.ThriftMux import com.twitter.finagle.thriftmux.MethodBuilder import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.inject.Injector import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.tweetconvosvc.thriftscala.ConversationService import com.twitter.util.Duration import org.apache.thrift.protocol.TCompactProtocol object ConversationServiceModule extends ThriftMethodBuilderClientModule[ ConversationService.ServicePerEndpoint, ConversationService.MethodPerEndpoint ] with MtlsClient { override val label: String = "tweetconvosvc" override val dest: String = "/s/tweetconvosvc/tweetconvosvc" override protected def configureMethodBuilder( injector: Injector, methodBuilder: MethodBuilder ): MethodBuilder = methodBuilder.withTimeoutPerRequest(100.milliseconds) override def configureThriftMuxClient( injector: Injector, client: ThriftMux.Client ): ThriftMux.Client = super .configureThriftMuxClient(injector, client) .withProtocolFactory(new TCompactProtocol.Factory()) override protected def sessionAcquisitionTimeout: Duration = 500.milliseconds } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/EarlybirdRealtimeCGModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.conversions.PercentOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.thrift.ClientId import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRealtimCGEndpoint import com.twitter.inject.TwitterModule import com.twitter.product_mixer.shared_library.thrift_client.FinagleThriftClientBuilder import com.twitter.product_mixer.shared_library.thrift_client.Idempotent import com.twitter.search.earlybird.{thriftscala => t} import javax.inject.Named import javax.inject.Singleton import org.apache.thrift.protocol.TCompactProtocol object EarlybirdRealtimeCGModule extends TwitterModule { val Label: String = "earlybird-rootrealtimecg" val Dest: String = "/s/earlybird-rootrealtimecg/root-realtime_cg" @Provides @Singleton @Named(EarlybirdRealtimCGEndpoint) def providesEarlybirdRealtimeCGService( serviceIdentifier: ServiceIdentifier, clientId: ClientId, statsReceiver: StatsReceiver ): t.EarlybirdService.MethodPerEndpoint = { FinagleThriftClientBuilder.buildFinagleMethodPerEndpoint[ t.EarlybirdService.ServicePerEndpoint, t.EarlybirdService.MethodPerEndpoint ]( serviceIdentifier = serviceIdentifier, clientId = clientId, dest = Dest, label = Label, statsReceiver = statsReceiver, protocolFactoryOverride = Some(new TCompactProtocol.Factory), idempotency = Idempotent(1.percent), timeoutPerRequest = 600.milliseconds, timeoutTotal = 650.milliseconds, acquisitionTimeout = 1.seconds ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/EventsRecosClientModule.scala ================================================ package com.twitter.home_mixer.module import com.twitter.conversions.DurationOps._ import com.twitter.events.recos.thriftscala.EventsRecosService import com.twitter.finagle.thriftmux.MethodBuilder import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.inject.Injector import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.util.Duration object EventsRecosClientModule extends ThriftMethodBuilderClientModule[ EventsRecosService.ServicePerEndpoint, EventsRecosService.MethodPerEndpoint ] with MtlsClient { override val label: String = "events-recos" override val dest: String = "/s/events-recos/events-recos-service" override protected def sessionAcquisitionTimeout: Duration = 500.milliseconds override protected def configureMethodBuilder( injector: Injector, methodBuilder: MethodBuilder ): MethodBuilder = { methodBuilder .withTimeoutPerRequest(450.millis) .withTimeoutTotal(450.millis) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/FeedbackHistoryClientModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.inject.annotations.Flag import com.twitter.timelinemixer.clients.feedback.FeedbackHistoryManhattanClient import com.twitter.timelinemixer.clients.feedback.FeedbackHistoryManhattanClientConfig import com.twitter.timelines.clients.manhattan.mhv3.ManhattanClientBuilder import com.twitter.util.Duration import javax.inject.Singleton object FeedbackHistoryClientModule extends TwitterModule { private val ProdDataset = "feedback_history" private val StagingDataset = "feedback_history_nonprod" private final val Timeout = "mh_feedback_history.timeout" flag[Duration](Timeout, 150.millis, "Timeout per request") @Provides @Singleton def providesFeedbackHistoryClient( @Flag(Timeout) timeout: Duration, serviceId: ServiceIdentifier, statsReceiver: StatsReceiver ) = { val manhattanDataset = serviceId.environment.toLowerCase match { case "prod" => ProdDataset case _ => StagingDataset } val config = new FeedbackHistoryManhattanClientConfig { val dataset = manhattanDataset val isReadOnly = true val serviceIdentifier = serviceId override val defaultMaxTimeout = timeout } new FeedbackHistoryManhattanClient( ManhattanClientBuilder.buildManhattanEndpoint(config, statsReceiver), manhattanDataset, statsReceiver ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/GizmoduckTimelinesCacheClientModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps.RichDuration import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.gizmoduck.{thriftscala => gt} import com.twitter.home_mixer.param.HomeMixerInjectionNames.GizmoduckTimelinesCache import com.twitter.inject.TwitterModule import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder import com.twitter.servo.cache.FinagleMemcache import com.twitter.servo.cache.KeyValueTransformingTtlCache import com.twitter.servo.cache.ObservableTtlCache import com.twitter.servo.cache.Serializer import com.twitter.servo.cache.ThriftSerializer import com.twitter.servo.cache.TtlCache import javax.inject.Named import javax.inject.Singleton import org.apache.thrift.protocol.TCompactProtocol import com.twitter.finagle.memcached.compressing.scheme.Lz4 object GizmoduckTimelinesCacheClientModule extends TwitterModule { private val ScopeName = "GizmoduckTimelinesCache" private val ProdDest = "/srv#/prod/local/cache/timelines_gizmoduck_secure:twemcaches" private val userSerializer: Serializer[gt.User] = { new ThriftSerializer[gt.User](gt.User, new TCompactProtocol.Factory()) } @Provides @Singleton @Named(GizmoduckTimelinesCache) def providesGizmoduckTimelinesCache( statsReceiver: StatsReceiver, serviceIdentifier: ServiceIdentifier ): TtlCache[Long, gt.User] = { val memCacheClient = MemcachedClientBuilder.buildMemcachedClient( destName = ProdDest, numTries = 1, numConnections = 1, requestTimeout = 100.milliseconds, globalTimeout = 100.milliseconds, connectTimeout = 100.milliseconds, acquisitionTimeout = 100.milliseconds, serviceIdentifier = serviceIdentifier, statsReceiver = statsReceiver, compressionScheme = Lz4 ) mkCache(new FinagleMemcache(memCacheClient), statsReceiver) } private def mkCache( finagleMemcache: FinagleMemcache, statsReceiver: StatsReceiver ): TtlCache[Long, gt.User] = { val baseCache: KeyValueTransformingTtlCache[Long, String, gt.User, Array[Byte]] = new KeyValueTransformingTtlCache( underlyingCache = finagleMemcache, transformer = userSerializer, underlyingKey = { key: Long => key.toString } ) ObservableTtlCache( underlyingCache = baseCache, statsReceiver = statsReceiver.scope(ScopeName), windowSize = 1000, name = ScopeName ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeAdsCandidateSourceModule.scala ================================================ package com.twitter.home_mixer.module import com.twitter.adserver.thriftscala.NewAdServer import com.twitter.conversions.DurationOps._ import com.twitter.finagle.thriftmux.MethodBuilder import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.inject.Injector import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.util.Duration object HomeAdsCandidateSourceModule extends ThriftMethodBuilderClientModule[ NewAdServer.ServicePerEndpoint, NewAdServer.MethodPerEndpoint ] with MtlsClient { override val label = "adserver" override val dest = "/s/ads/adserver" override protected def configureMethodBuilder( injector: Injector, methodBuilder: MethodBuilder ): MethodBuilder = { methodBuilder .withTimeoutPerRequest(1200.milliseconds) .withTimeoutTotal(1200.milliseconds) .withMaxRetries(2) } override protected def sessionAcquisitionTimeout: Duration = 150.milliseconds } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFeaturesModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.conversions.PercentOps._ import com.twitter.finagle.ThriftMux import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.thrift.ClientId import com.twitter.finagle.thrift.RichClientParam import com.twitter.inject.TwitterModule import com.twitter.home_mixer_features.{thriftjava => tj} import com.twitter.home_mixer_features.{thriftscala => ts} import com.twitter.product_mixer.shared_library.thrift_client.FinagleThriftClientBuilder import com.twitter.product_mixer.shared_library.thrift_client.Idempotent import javax.inject.Singleton object HomeMixerFeaturesModule extends TwitterModule { val Label: String = "home-mixer-features" val Dest: String = "/s/home-mixer/home-mixer-features" @Provides @Singleton def providesHomeMixerFeaturesService( serviceIdentifier: ServiceIdentifier, clientId: ClientId, statsReceiver: StatsReceiver, ): tj.HomeMixerFeatures.ServiceToClient = { buildClient(serviceIdentifier, clientId, statsReceiver, Dest, Label) } @Provides @Singleton def providesHomeMixerFeaturesScalaService( serviceIdentifier: ServiceIdentifier, clientId: ClientId, statsReceiver: StatsReceiver, ): ts.HomeMixerFeatures.MethodPerEndpoint = { FinagleThriftClientBuilder.buildFinagleMethodPerEndpoint[ ts.HomeMixerFeatures.ServicePerEndpoint, ts.HomeMixerFeatures.MethodPerEndpoint ]( serviceIdentifier = serviceIdentifier, clientId = clientId, dest = Dest, label = Label, statsReceiver = statsReceiver, idempotency = Idempotent(1.percent), timeoutPerRequest = 300.milliseconds, timeoutTotal = 300.milliseconds, acquisitionTimeout = 1.seconds ) } private def buildClient( serviceIdentifier: ServiceIdentifier, clientId: ClientId, statsReceiver: StatsReceiver, dest: String, label: String ): tj.HomeMixerFeatures.ServiceToClient = { val stats = statsReceiver.scope("clnt") val thriftClient = ThriftMux.client .withMutualTls(serviceIdentifier) .withClientId(clientId) .withLabel(label) .withStatsReceiver(stats) .withRequestTimeout(300.milliseconds) .withSession.acquisitionTimeout(1.second) .methodBuilder(dest) .withTimeoutTotal(300.milliseconds) .idempotent(1.percent) .newService new tj.HomeMixerFeatures.ServiceToClient( thriftClient, RichClientParam() ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerFlagsModule.scala ================================================ package com.twitter.home_mixer.module import com.twitter.conversions.DurationOps.RichDuration import com.twitter.home_mixer.param.HomeMixerFlagName import com.twitter.inject.TwitterModule import com.twitter.util.Duration object HomeMixerFlagsModule extends TwitterModule { import HomeMixerFlagName._ flag[Boolean]( name = ScribeClientEventsFlag, default = false, help = "Toggles logging client events to Scribe" ) flag[Boolean]( name = ScribeServedCandidatesFlag, default = false, help = "Toggles logging served candidates to Scribe" ) flag[Boolean]( name = ScribeScoredCandidatesFlag, default = false, help = "Toggles logging scored candidates to Scribe" ) flag[Boolean]( name = ScribeFeaturesFlag, default = false, help = "Toggles logging served common features and candidates features to Scribe" ) flag[String]( name = DataRecordMetadataStoreConfigsYmlFlag, default = "mysql_timelines_ro_prod.yml", help = "The YML file that contains the necessary info for creating metadata store MySQL client." ) flag[String]( name = DarkTrafficFilterDeciderKey, default = "dark_traffic_filter", help = "Dark traffic filter decider key" ) flag[Duration]( TargetScoringLatency, 700.millis, "Target scoring latency for Quality Factor" ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/HomeMixerResourcesModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.config.yaml.YamlMap import com.twitter.inject.TwitterModule import javax.inject.Singleton case class ViralContentCreatorsConfig(creators: Set[Long]) case class SupportAccountsConfig(accounts: Set[Long]) object HomeMixerResourcesModule extends TwitterModule { private val ConfigFilePath = "/config/ids.yml" private val SupportAccountsKey = "support_accounts" private val ViralContentCreatorsKey = "viral_content_creators" private val yaml: YamlMap = YamlMap.load(ConfigFilePath) @Singleton @Provides def providesViralContentCreatorsConfig: ViralContentCreatorsConfig = { val contentCreators = yaml.longSeq(ViralContentCreatorsKey).toSet ViralContentCreatorsConfig(contentCreators) } @Singleton @Provides def providesSupportAccountsConfig: SupportAccountsConfig = { val supportAccounts = yaml.longSeq(SupportAccountsKey).toSet SupportAccountsConfig(supportAccounts) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ImpressionBloomFilterModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.inject.annotations.Flag import com.twitter.storage.client.manhattan.kv.Guarantee import com.twitter.storehaus_internal.manhattan.ManhattanClusters import com.twitter.timelines.clients.manhattan.store._ import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm} import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilterManhattanKeyValueDescriptor import com.twitter.util.Duration import javax.inject.Singleton object ImpressionBloomFilterModule extends TwitterModule { private val ProdAppId = "impression_bloom_filter_store" private val ProdDataset = "impression_bloom_filter" private val StagingAppId = "impression_bloom_filter_store_staging" private val StagingDataset = "impression_bloom_filter_staging" private val ClientStatsScope = "tweetBloomFilterImpressionManhattanClient" private val DefaultTTL = 7.days private final val Timeout = "mh_impression_store_bloom_filter.timeout" flag[Duration](Timeout, 150.millis, "Timeout per request") @Provides @Singleton def providesImpressionBloomFilter( @Flag(Timeout) timeout: Duration, serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): ManhattanStoreClient[blm.ImpressionBloomFilterKey, blm.ImpressionBloomFilterSeq] = { val (appId, dataset) = serviceIdentifier.environment.toLowerCase match { case "prod" => (ProdAppId, ProdDataset) case _ => (StagingAppId, StagingDataset) } implicit val manhattanKeyValueDescriptor: ImpressionBloomFilterManhattanKeyValueDescriptor = ImpressionBloomFilterManhattanKeyValueDescriptor( dataset = dataset, ttl = DefaultTTL ) ManhattanStoreClientBuilder.buildManhattanClient( serviceIdentifier = serviceIdentifier, cluster = ManhattanClusters.nash, appId = appId, defaultMaxTimeout = timeout, maxRetryCount = 2, defaultGuarantee = Some(Guarantee.SoftDcReadMyWrites), isReadOnly = false, statsScope = ClientStatsScope, statsReceiver = statsReceiver ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/InMemoryCacheModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.home_mixer.param.HomeMixerInjectionNames.ImageClipClusterIdInMemCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.IsColdStartPostInMemCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClipClusterIdInMemCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaCompletionRateInMemCache import com.twitter.inject.TwitterModule import com.twitter.servo.cache.ExpiringLruInProcessCache import com.twitter.servo.cache.InProcessCache import javax.inject.Named import javax.inject.Singleton import scala.util.Random object InMemoryCacheModule extends TwitterModule { @Singleton @Provides @Named(MediaClipClusterIdInMemCache) def providesMediaClipClusterIdInMemCache( ): InProcessCache[Long, Option[Option[Long]]] = { val BaseTTL = 4 val TTL = (BaseTTL + Random.nextInt(3)).minutes val cache: InProcessCache[Long, Option[Option[Long]]] = new ExpiringLruInProcessCache(ttl = TTL, maximumSize = 500000) cache } @Singleton @Provides @Named(ImageClipClusterIdInMemCache) def providesImageClipClusterIdInMemCache( ): InProcessCache[Long, Option[Option[Long]]] = { val BaseTTL = 4 val TTL = (BaseTTL + Random.nextInt(3)).minutes val cache: InProcessCache[Long, Option[Option[Long]]] = new ExpiringLruInProcessCache(ttl = TTL, maximumSize = 500000) cache } @Singleton @Provides @Named(MediaCompletionRateInMemCache) def providesMediaCompletionRateInMemCache( ): InProcessCache[Long, Double] = { val BaseTTL = 20 val TTL = (BaseTTL + Random.nextInt(15)).minutes val cache: InProcessCache[Long, Double] = new ExpiringLruInProcessCache(ttl = TTL, maximumSize = 10000000) cache } @Singleton @Provides @Named(IsColdStartPostInMemCache) def providesIsColdStartPostInMemCache( ): InProcessCache[Long, Boolean] = { val BaseTTL = 4 val TTL = (BaseTTL + Random.nextInt(3)).minutes val cache: InProcessCache[Long, Boolean] = new ExpiringLruInProcessCache(ttl = TTL, maximumSize = 500000) cache } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/InjectionHistoryClientModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.ThriftMux import com.twitter.finagle.builder.ClientBuilder import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.mtls.client.MtlsStackClient._ import com.twitter.finagle.service.RetryPolicy import com.twitter.finagle.ssl.OpportunisticTls import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.manhattan.v2.thriftscala.{ManhattanCoordinator => ManhattanV2} import com.twitter.timelinemixer.clients.manhattan.InjectionHistoryClient import com.twitter.timelinemixer.clients.manhattan.ManhattanDatasetConfig import com.twitter.timelines.clients.manhattan.Dataset import com.twitter.timelines.clients.manhattan.ManhattanClient import com.twitter.timelines.util.stats.RequestScope import javax.inject.Singleton import org.apache.thrift.protocol.TBinaryProtocol import com.twitter.timelines.config.TimelinesUnderlyingClientConfiguration.ConnectTimeout import com.twitter.timelines.config.TimelinesUnderlyingClientConfiguration.TCPConnectTimeout object InjectionHistoryClientModule extends TwitterModule { private val ProdDataset = "suggestion_history" private val StagingDataset = "suggestion_history_nonprod" private val AppId = "twitter_suggests" private val ServiceName = "manhattan.omega" private val OmegaManhattanDest = "/s/manhattan/omega.native-thrift" private val InjectionRequestScope = RequestScope("injectionHistoryClient") private val RequestTimeout = 75.millis private val Timeout = 150.millis val retryPolicy = RetryPolicy.tries( 2, RetryPolicy.TimeoutAndWriteExceptionsOnly .orElse(RetryPolicy.ChannelClosedExceptionsOnly)) @Provides @Singleton def providesInjectionHistoryClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ) = { val dataset = serviceIdentifier.environment.toLowerCase match { case "prod" => ProdDataset case _ => StagingDataset } val thriftMuxClient = ClientBuilder() .name(ServiceName) .daemon(daemonize = true) .failFast(enabled = true) .retryPolicy(retryPolicy) .tcpConnectTimeout(TCPConnectTimeout) .connectTimeout(ConnectTimeout) .dest(OmegaManhattanDest) .requestTimeout(RequestTimeout) .timeout(Timeout) .stack(ThriftMux.client .withMutualTls(serviceIdentifier) .withOpportunisticTls(OpportunisticTls.Required)) .build() val manhattanOmegaClient = new ManhattanV2.FinagledClient( service = thriftMuxClient, protocolFactory = new TBinaryProtocol.Factory(), serviceName = ServiceName, ) val readOnlyMhClient = new ManhattanClient( appId = AppId, manhattan = manhattanOmegaClient, requestScope = InjectionRequestScope, serviceName = ServiceName, statsReceiver = statsReceiver ).readOnly val mhDatasetConfig = new ManhattanDatasetConfig { override val SuggestionHistoryDataset = Dataset(dataset) } new InjectionHistoryClient( readOnlyMhClient, mhDatasetConfig ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/LimiterModule.scala ================================================ package com.twitter.home_mixer.module import com.twitter.conversions.DurationOps._ import com.twitter.finagle.thriftmux.MethodBuilder import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.inject.Injector import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.limiter.{thriftscala => t} import com.twitter.util.Duration object LimiterModule extends ThriftMethodBuilderClientModule[ t.LimitService.ServicePerEndpoint, t.LimitService.MethodPerEndpoint ] with MtlsClient { override val label: String = "limiter" override val dest: String = "/s/limiter/limiter" override protected def configureMethodBuilder( injector: Injector, methodBuilder: MethodBuilder ): MethodBuilder = methodBuilder.withTimeoutPerRequest(200.milliseconds) override protected def sessionAcquisitionTimeout: Duration = 500.milliseconds } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanClientsModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.home_mixer.param.HomeMixerInjectionNames.RTAManhattanEndpoint import com.twitter.home_mixer.param.HomeMixerInjectionNames.RTAManhattanStore import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphManhattanEndpoint import com.twitter.home_mixer.store.RTAMHStore import com.twitter.inject.TwitterModule import com.twitter.inject.annotations.Flag import com.twitter.storage.client.manhattan.kv._ import com.twitter.timelines.config.ConfigUtils import com.twitter.util.Duration import com.twitter.timelines.realtime_aggregates.{thriftscala => thrift} import com.twitter.ml.api.DataRecord import com.twitter.storehaus.ReadableStore import javax.inject.Named import javax.inject.Singleton object ManhattanClientsModule extends TwitterModule with ConfigUtils { private val ApolloDest = "/s/manhattan/apollo.native-thrift" private val BalterDest = "/s/manhattan/baltar.native-thrift" private final val Timeout = "mh_real_graph.timeout" flag[Duration](Timeout, 150.millis, "Timeout total") @Provides @Singleton @Named(RealGraphManhattanEndpoint) def providesRealGraphManhattanEndpoint( @Flag(Timeout) timeout: Duration, serviceIdentifier: ServiceIdentifier ): ManhattanKVEndpoint = { lazy val client = ManhattanKVClient( appId = "real_graph", dest = ApolloDest, mtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier = serviceIdentifier), label = "real-graph-data" ) ManhattanKVEndpointBuilder(client) .maxRetryCount(2) .defaultMaxTimeout(timeout) .build() } @Provides @Singleton @Named(RTAManhattanEndpoint) def providesRTAManhattanEndpoint( @Flag(Timeout) timeout: Duration, serviceIdentifier: ServiceIdentifier ): ManhattanKVEndpoint = { lazy val client = ManhattanKVClient( appId = "timelines_real_time_aggregates", dest = BalterDest, mtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier = serviceIdentifier), label = "rta-test" ) ManhattanKVEndpointBuilder(client) .maxRetryCount(2) .defaultMaxTimeout(timeout) .build() } @Provides @Singleton @Named(RTAManhattanStore) def providesRTAManhattanStore( @Named(RTAManhattanEndpoint) manhattanKVEndpoint: ManhattanKVEndpoint ): Option[ReadableStore[thrift.AggregationKey, DataRecord]] = { Some(new RTAMHStore(manhattanKVEndpoint)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanFeatureRepositoryModule.scala ================================================ package com.twitter.home_mixer.module import com.google.common.primitives.Longs import com.google.inject.Provides import com.twitter.bijection.Injection import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.bijection.scrooge.CompactScalaCodec import com.twitter.bijection.thrift.ThriftCodec import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.home_mixer.param.HomeMixerInjectionNames._ import com.twitter.home_mixer.util.InjectionTransformerImplicits._ import com.twitter.home_mixer.util.LanguageUtil import com.twitter.home_mixer.util.TensorFlowUtil import com.twitter.inject.TwitterModule import com.twitter.manhattan.v1.{thriftscala => mh} import com.twitter.ml.api.{thriftscala => ml} import com.twitter.ml.featurestore.lib.UserId import com.twitter.ml.featurestore.{thriftscala => fs} import com.twitter.product_mixer.shared_library.manhattan_client.ManhattanClientBuilder import com.twitter.scalding_internal.multiformat.format.keyval.KeyValInjection.ScalaBinaryThrift import com.twitter.search.common.constants.{thriftscala => scc} import com.twitter.service.metastore.gen.{thriftscala => smg} import com.twitter.servo.cache._ import com.twitter.servo.manhattan.ManhattanKeyValueRepository import com.twitter.servo.repository.CachingKeyValueRepository import com.twitter.servo.repository.ChunkingStrategy import com.twitter.servo.repository.KeyValueRepository import com.twitter.servo.repository.Repository import com.twitter.servo.repository.keysAsQuery import com.twitter.servo.util.Transformer import com.twitter.storage.client.manhattan.bijections.Bijections import com.twitter.storehaus_internal.manhattan.ManhattanClusters import com.twitter.timelines.author_features.v1.{thriftjava => af} import com.twitter.timelines.suggests.common.dense_data_record.{thriftscala => ddr} import com.twitter.user_session_store.{thriftjava => uss} import com.twitter.user_session_store.{thriftscala => uss_scala} import com.twitter.util.Duration import com.twitter.util.Try import java.nio.ByteBuffer import javax.inject.Named import javax.inject.Singleton import org.apache.thrift.protocol.TCompactProtocol import org.apache.thrift.transport.TMemoryInputTransport import org.apache.thrift.transport.TTransport object ManhattanFeatureRepositoryModule extends TwitterModule { private val DEFAULT_RPC_CHUNK_SIZE = 50 private val ThriftEntityIdInjection = ScalaBinaryThrift(fs.EntityId) private val FeatureStoreUserIdKeyTransformer = new Transformer[Long, ByteBuffer] { override def to(userId: Long): Try[ByteBuffer] = { Try(ByteBuffer.wrap(ThriftEntityIdInjection.apply(UserId(userId).toThrift))) } override def from(b: ByteBuffer): Try[Long] = ??? } private val LongUserIdKeyTransformer = new Transformer[Long, ByteBuffer] { override def to(userId: Long): Try[ByteBuffer] = { Try(ByteBuffer.wrap(Longs.toByteArray(userId))) } override def from(b: ByteBuffer): Try[Long] = ??? } private val FloatTensorTransformer = new Transformer[ByteBuffer, ml.FloatTensor] { override def to(input: ByteBuffer): Try[ml.FloatTensor] = { val floatTensor = TensorFlowUtil.embeddingByteBufferToFloatTensor(input) Try(floatTensor) } override def from(b: ml.FloatTensor): Try[ByteBuffer] = ??? } private val EmbeddingTransformer = new Transformer[ByteBuffer, ml.FloatTensor] { override def to(input: ByteBuffer): Try[ml.FloatTensor] = { Try.fromScala( Bijections .BinaryScalaInjection(ml.Embedding).andThen(Bijections.byteBuffer2Buf.inverse).invert( input).map { embedding => embedding.tensor.map { tensor => TensorFlowUtil.embeddingNoHeaderByteBufferToFloatTensor(tensor.content) }.get } ) } override def from(b: ml.FloatTensor): Try[ByteBuffer] = ??? } private val LanguageTransformer = new Transformer[ByteBuffer, Seq[scc.ThriftLanguage]] { override def to(input: ByteBuffer): Try[Seq[scc.ThriftLanguage]] = { Try.fromScala( Bijections .BinaryScalaInjection(smg.UserLanguages) .andThen(Bijections.byteBuffer2Buf.inverse) .invert(input).map(LanguageUtil.computeLanguages(_))) } override def from(b: Seq[scc.ThriftLanguage]): Try[ByteBuffer] = ??? } private val LongKeyTransformer = Injection .connect[Long, Array[Byte]] .toByteBufferTransformer() // manhattan clients @Provides @Singleton @Named(ManhattanApolloClient) def providesManhattanApolloClient( serviceIdentifier: ServiceIdentifier ): mh.ManhattanCoordinator.MethodPerEndpoint = { ManhattanClientBuilder .buildManhattanV1FinagleClient( ManhattanClusters.apollo, serviceIdentifier ) } @Provides @Singleton @Named(ManhattanAthenaClient) def providesManhattanAthenaClient( serviceIdentifier: ServiceIdentifier ): mh.ManhattanCoordinator.MethodPerEndpoint = { ManhattanClientBuilder .buildManhattanV1FinagleClient( ManhattanClusters.athena, serviceIdentifier ) } @Provides @Singleton @Named(ManhattanOmegaClient) def providesManhattanOmegaClient( serviceIdentifier: ServiceIdentifier ): mh.ManhattanCoordinator.MethodPerEndpoint = { ManhattanClientBuilder .buildManhattanV1FinagleClient( ManhattanClusters.omega, serviceIdentifier ) } @Provides @Singleton @Named(ManhattanStarbuckClient) def providesManhattanStarbuckClient( serviceIdentifier: ServiceIdentifier ): mh.ManhattanCoordinator.MethodPerEndpoint = { ManhattanClientBuilder .buildManhattanV1FinagleClient( ManhattanClusters.starbuck, serviceIdentifier ) } // non-cached manhattan repositories /** * A repository of the offline aggregate feature metadata necessary to decode * DenseCompactDataRecords. * * This repository is expected to virtually always pick up the metadata form the local cache with * nearly 0 latency. */ @Provides @Singleton @Named(TimelineAggregateMetadataRepository) def providesTimelineAggregateMetadataRepository( @Named(ManhattanAthenaClient) client: mh.ManhattanCoordinator.MethodPerEndpoint ): Repository[Int, Option[ddr.DenseFeatureMetadata]] = { val keyTransformer = Injection .connect[Int, Array[Byte]] .toByteBufferTransformer() val valueTransformer = new Transformer[ByteBuffer, ddr.DenseFeatureMetadata] { private val compactProtocolFactory = new TCompactProtocol.Factory def to(buffer: ByteBuffer): Try[ddr.DenseFeatureMetadata] = Try { val transport = transportFromByteBuffer(buffer) ddr.DenseFeatureMetadata.decode(compactProtocolFactory.getProtocol(transport)) } // Encoding intentionally not implemented as it is never used def from(metadata: ddr.DenseFeatureMetadata): Try[ByteBuffer] = ??? } val inProcessCache: Cache[Int, Cached[ddr.DenseFeatureMetadata]] = InProcessLruCacheFactory( ttl = Duration.fromMinutes(20), lruSize = 30 ).apply(serializer = Transformer(_ => ???, _ => ???)) // Serialization is not necessary here. val keyValueRepository = new ManhattanKeyValueRepository( client = client, keyTransformer = keyTransformer, valueTransformer = valueTransformer, appId = "timelines_dense_aggregates_encoding_metadata", // Expected QPS is negligible. dataset = "user_session_dense_feature_metadata", timeoutInMillis = 100 ) KeyValueRepository .singular( new CachingKeyValueRepository[Seq[Int], Int, ddr.DenseFeatureMetadata]( keyValueRepository, new NonLockingCache(inProcessCache), keysAsQuery[Int] ) ) } @Provides @Singleton @Named(RealGraphFeatureRepository) def providesRealGraphFeatureRepository( @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint ): Repository[Long, Option[uss_scala.UserSession]] = { val valueTransformer = CompactScalaCodec(uss_scala.UserSession).toByteBufferTransformer().flip KeyValueRepository.singular( new ManhattanKeyValueRepository( client = client, keyTransformer = LongKeyTransformer, valueTransformer = valueTransformer, appId = "real_graph", dataset = "real_graph_user_features", timeoutInMillis = 100, ) ) } // cached manhattan repositories @Provides @Singleton @Named(AuthorFeatureRepository) def providesAuthorFeatureRepository( @Named(ManhattanAthenaClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, @Named(HomeAuthorFeaturesCacheClient) cacheClient: Memcache ): KeyValueRepository[Seq[Long], Long, af.AuthorFeatures] = { val valueInjection = ThriftCodec .toCompact[af.AuthorFeatures] val keyValueRepository = batchedManhattanKeyValueRepository( client = client, keyTransformer = LongKeyTransformer, valueTransformer = valueInjection.toByteBufferTransformer().flip, appId = "timelines_author_feature_store_athena", dataset = "timelines_author_features", timeoutInMillis = 100 ) val remoteCacheRepo = buildMemCachedRepository( keyValueRepository = keyValueRepository, cacheClient = cacheClient, cachePrefix = "AuthorFeatureHydrator", ttl = 12.hours, valueInjection = valueInjection ) remoteCacheRepo } @Provides @Singleton @Named(TwhinAuthorFollowFeatureRepository) def providesTwhinAuthorFollowFeatureRepository( @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, @Named(TwhinAuthorFollowFeatureCacheClient) cacheClient: Memcache ): KeyValueRepository[Seq[Long], Long, ml.FloatTensor] = { val keyValueRepository = batchedManhattanKeyValueRepository( client = client, keyTransformer = FeatureStoreUserIdKeyTransformer, valueTransformer = FloatTensorTransformer, appId = "ml_features_apollo", dataset = "twhin_author_follow_embedding_fsv1__v1_thrift__embedding", timeoutInMillis = 100 ) val valueInjection: Injection[ml.FloatTensor, Array[Byte]] = BinaryScalaCodec(ml.FloatTensor) buildMemCachedRepository( keyValueRepository = keyValueRepository, cacheClient = cacheClient, cachePrefix = "twhinAuthorFollows", ttl = 24.hours, valueInjection = valueInjection ) } @Provides @Singleton @Named(UserLanguagesRepository) def providesUserLanguagesFeatureRepository( @Named(ManhattanStarbuckClient) client: mh.ManhattanCoordinator.MethodPerEndpoint ): KeyValueRepository[Seq[Long], Long, Seq[scc.ThriftLanguage]] = { batchedManhattanKeyValueRepository( client = client, keyTransformer = LongKeyTransformer, valueTransformer = LanguageTransformer, appId = "user_metadata", dataset = "languages", timeoutInMillis = 70 ) } @Provides @Singleton @Named(TwhinUserFollowFeatureRepository) def providesTwhinUserFollowFeatureRepository( @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint ): KeyValueRepository[Seq[Long], Long, ml.FloatTensor] = { batchedManhattanKeyValueRepository( client = client, keyTransformer = FeatureStoreUserIdKeyTransformer, valueTransformer = FloatTensorTransformer, appId = "ml_features_apollo", dataset = "twhin_user_follow_embedding_fsv1__v1_thrift__embedding", timeoutInMillis = 100 ) } @Provides @Singleton @Named(TimelineAggregatePartARepository) def providesTimelineAggregatePartARepository( @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, ): Repository[Long, Option[uss.UserSession]] = timelineAggregateRepository( mhClient = client, mhDataset = "timelines_aggregates_v2_features_by_user_part_a_apollo", mhAppId = "timelines_aggregates_v2_features_by_user_part_a_apollo" ) @Provides @Singleton @Named(TimelineAggregatePartBRepository) def providesTimelineAggregatePartBRepository( @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint, ): Repository[Long, Option[uss.UserSession]] = timelineAggregateRepository( mhClient = client, mhDataset = "timelines_aggregates_v2_features_by_user_part_b_apollo", mhAppId = "timelines_aggregates_v2_features_by_user_part_b_apollo" ) @Provides @Singleton @Named(TwhinUserEngagementFeatureRepository) def providesTwhinUserEngagementFeatureRepository( @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint ): KeyValueRepository[Seq[Long], Long, ml.FloatTensor] = { batchedManhattanKeyValueRepository( client = client, keyTransformer = FeatureStoreUserIdKeyTransformer, valueTransformer = FloatTensorTransformer, appId = "ml_features_apollo", dataset = "twhin_user_engagement_embedding_fsv1__v1_thrift__embedding", timeoutInMillis = 100 ) } @Provides @Singleton @Named(TwhinRebuildUserEngagementFeatureRepository) def providesTwhinRebuildUserEngagementFeatureRepository( @Named(ManhattanApolloClient) client: mh.ManhattanCoordinator.MethodPerEndpoint ): KeyValueRepository[Seq[Long], Long, ml.FloatTensor] = { batchedManhattanKeyValueRepository( client = client, keyTransformer = LongUserIdKeyTransformer, valueTransformer = EmbeddingTransformer, appId = "twhin_embeddings_apollo", dataset = "twhin_refreshed_user_eng_emb", timeoutInMillis = 100 ) } private def buildMemCachedRepository[K, V]( keyValueRepository: KeyValueRepository[Seq[K], K, V], cacheClient: Memcache, cachePrefix: String, ttl: Duration, valueInjection: Injection[V, Array[Byte]] ): CachingKeyValueRepository[Seq[K], K, V] = { val cachedSerializer = CachedSerializer.binary( valueInjection.toByteArrayTransformer() ) val cache = MemcacheCacheFactory( cacheClient, ttl, PrefixKeyTransformerFactory(cachePrefix) )[K, Cached[V]](cachedSerializer) new CachingKeyValueRepository( keyValueRepository, new NonLockingCache(cache), keysAsQuery[K] ) } private def buildInProcessCachedRepository[K, V]( keyValueRepository: KeyValueRepository[Seq[K], K, V], ttl: Duration, size: Int, valueInjection: Injection[V, Array[Byte]] ): CachingKeyValueRepository[Seq[K], K, V] = { val cachedSerializer = CachedSerializer.binary( valueInjection.toByteArrayTransformer() ) val cache = InProcessLruCacheFactory( ttl = ttl, lruSize = size )[K, Cached[V]](cachedSerializer) new CachingKeyValueRepository( keyValueRepository, new NonLockingCache(cache), keysAsQuery[K] ) } private def batchedManhattanKeyValueRepository[K, V]( client: mh.ManhattanCoordinator.MethodPerEndpoint, keyTransformer: Transformer[K, ByteBuffer], valueTransformer: Transformer[ByteBuffer, V], appId: String, dataset: String, timeoutInMillis: Int, chunkSize: Int = DEFAULT_RPC_CHUNK_SIZE ): KeyValueRepository[Seq[K], K, V] = KeyValueRepository.chunked( new ManhattanKeyValueRepository( client = client, keyTransformer = keyTransformer, valueTransformer = valueTransformer, appId = appId, dataset = dataset, timeoutInMillis = timeoutInMillis ), chunker = ChunkingStrategy.equalSize(chunkSize) ) private def transportFromByteBuffer(buffer: ByteBuffer): TTransport = new TMemoryInputTransport( buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()) private def timelineAggregateRepository( mhClient: mh.ManhattanCoordinator.MethodPerEndpoint, mhDataset: String, mhAppId: String ): Repository[Long, Option[uss.UserSession]] = { val valueInjection = ThriftCodec .toCompact[uss.UserSession] KeyValueRepository.singular( new ManhattanKeyValueRepository( client = mhClient, keyTransformer = LongKeyTransformer, valueTransformer = valueInjection.toByteBufferTransformer().flip, appId = mhAppId, dataset = mhDataset, timeoutInMillis = 100 ) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ManhattanTweetImpressionStoreModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.inject.annotations.Flag import com.twitter.storage.client.manhattan.kv.Guarantee import com.twitter.storehaus_internal.manhattan.ManhattanClusters import com.twitter.timelines.clients.manhattan.mhv3.ManhattanClientBuilder import com.twitter.timelines.impressionstore.store.ManhattanTweetImpressionStoreClientConfig import com.twitter.timelines.impressionstore.store.ManhattanTweetImpressionStoreClient import com.twitter.util.Duration import javax.inject.Singleton object ManhattanTweetImpressionStoreModule extends TwitterModule { private val ProdAppId = "timelines_tweet_impression_store_v2" private val ProdDataset = "timelines_tweet_impressions_v2" private val StagingAppId = "timelines_tweet_impression_store_staging" private val StagingDataset = "timelines_tweet_impressions_staging" private val StatsScope = "manhattanTweetImpressionStoreClient" private val DefaultTTL = 2.days private final val Timeout = "mh_impression_store.timeout" flag[Duration](Timeout, 150.millis, "Timeout per request") @Provides @Singleton def providesManhattanTweetImpressionStoreClient( @Flag(Timeout) timeout: Duration, serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): ManhattanTweetImpressionStoreClient = { val (appId, dataset) = serviceIdentifier.environment.toLowerCase match { case "prod" => (ProdAppId, ProdDataset) case _ => (StagingAppId, StagingDataset) } val config = ManhattanTweetImpressionStoreClientConfig( cluster = ManhattanClusters.nash, appId = appId, dataset = dataset, statsScope = StatsScope, defaultGuarantee = Guarantee.SoftDcReadMyWrites, defaultMaxTimeout = timeout, maxRetryCount = 2, isReadOnly = false, serviceIdentifier = serviceIdentifier, ttl = DefaultTTL ) val manhattanEndpoint = ManhattanClientBuilder.buildManhattanEndpoint(config, statsReceiver) ManhattanTweetImpressionStoreClient(config, manhattanEndpoint, statsReceiver) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MediaClusterId88Module.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClusterId88Store import com.twitter.home_mixer.store.MediaClusterId88Store import com.twitter.inject.TwitterModule import com.twitter.storehaus.ReadableStore import javax.inject.Named import javax.inject.Singleton object MediaClusterId88Module extends TwitterModule { @Provides @Singleton @Named(MediaClusterId88Store) def providesMediaClusterId88Store( mediaClusterId88Store: MediaClusterId88Store ): ReadableStore[Long, Long] = { mediaClusterId88Store.clusterIdStore } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MediaClusterIdModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.home_mixer.param.HomeMixerInjectionNames.MediaClusterId95Store import com.twitter.home_mixer.store.MediaClusterId95Store import com.twitter.inject.TwitterModule import com.twitter.storehaus.ReadableStore import javax.inject.Named import javax.inject.Singleton object MediaClusterId95Module extends TwitterModule { @Provides @Singleton @Named(MediaClusterId95Store) def providesMediaClusterId95Store( mediaClusterId95Store: MediaClusterId95Store ): ReadableStore[Long, Long] = { mediaClusterId95Store.clusterIdStore } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedFeatureRepositoryModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.Memcached import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.EntityRealGraphClientStore import com.twitter.home_mixer.param.HomeMixerInjectionNames.HomeAuthorFeaturesCacheClient import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionGraphUserVertexClient import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelinesRealTimeAggregateClient import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollowFeatureCacheClient import com.twitter.inject.TwitterModule import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder import com.twitter.servo.cache.ExpiringLruInProcessCache import com.twitter.servo.cache.FinagleMemcache import com.twitter.servo.cache.FinagleMemcacheFactory import com.twitter.servo.cache.HotKeyMemcacheClient import com.twitter.servo.cache.Memcache import com.twitter.storehaus.ReadableStore import com.twitter.wtf.entity_real_graph.summingbird.client.EntityRealGraphClient import com.twitter.wtf.entity_real_graph.summingbird.common.config.Configs.Environment import com.twitter.wtf.entity_real_graph.{thriftscala => erg} import com.twitter.finagle.memcached.compressing.scheme.Lz4 import com.twitter.home_mixer.param.HomeMixerInjectionNames.TvRealTimeAggregateClient import javax.inject.Named import javax.inject.Singleton object MemcachedFeatureRepositoryModule extends TwitterModule { // This must match the respective parameter on the write path. Note that servo sets a different // hasher by default. See [[com.twitter.hashing.KeyHasher]] for the list of other available // hashers. private val memcacheKeyHasher = "ketama" @Provides @Singleton @Named(TimelinesRealTimeAggregateClient) def providesTimelinesRealTimeAggregateClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Memcache = { val cacheClient = MemcachedClientBuilder.buildMemcachedClient( destName = "/s/cache/timelines_real_time_aggregates:twemcaches", numTries = 3, numConnections = 1, requestTimeout = 100.milliseconds, globalTimeout = 300.milliseconds, connectTimeout = 200.milliseconds, acquisitionTimeout = 200.milliseconds, serviceIdentifier = serviceIdentifier, statsReceiver = statsReceiver ) val hotkeyCacheClient = new HotKeyMemcacheClient( proxyClient = cacheClient, inProcessCache = new ExpiringLruInProcessCache(ttl = 15.minute, maximumSize = 75000), statsReceiver = statsReceiver.scope(TimelinesRealTimeAggregateClient).scope("inProcess") ) new FinagleMemcache(hotkeyCacheClient, memcacheKeyHasher) } @Provides @Singleton @Named(HomeAuthorFeaturesCacheClient) def providesHomeAuthorFeaturesCacheClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Memcache = { val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( numTries = 2, numConnections = 1, requestTimeout = 150.milliseconds, globalTimeout = 300.milliseconds, connectTimeout = 200.milliseconds, acquisitionTimeout = 200.milliseconds, serviceIdentifier = serviceIdentifier, statsReceiver = statsReceiver, compressionScheme = Lz4 ) buildMemcacheClient(cacheClient, "/s/cache/timelines_author_features:twemcaches") } @Provides @Singleton @Named(TwhinAuthorFollowFeatureCacheClient) def providesTwhinAuthorFollowFeatureCacheClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Memcache = { val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( numTries = 2, numConnections = 1, requestTimeout = 150.milliseconds, globalTimeout = 300.milliseconds, connectTimeout = 200.milliseconds, acquisitionTimeout = 200.milliseconds, serviceIdentifier = serviceIdentifier, statsReceiver = statsReceiver ) buildMemcacheClient(cacheClient, "/s/cache/home_twhin_author_features:twemcaches") } @Provides @Singleton @Named(RealTimeInteractionGraphUserVertexClient) def providesRealTimeInteractionGraphUserVertexClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Memcache = { val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( numTries = 2, numConnections = 1, requestTimeout = 150.milliseconds, globalTimeout = 300.milliseconds, connectTimeout = 200.milliseconds, acquisitionTimeout = 200.milliseconds, serviceIdentifier = serviceIdentifier, statsReceiver = statsReceiver ) buildMemcacheClient(cacheClient, "/s/cache/realtime_interactive_graph_prod_v2:twemcaches") } @Provides @Singleton @Named(TvRealTimeAggregateClient) def providesTvRealTimeAggregateClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Memcache = { val cacheClient = MemcachedClientBuilder.buildRawMemcachedClient( numTries = 1, numConnections = 1, requestTimeout = 200.milliseconds, globalTimeout = 200.milliseconds, connectTimeout = 200.milliseconds, acquisitionTimeout = 200.milliseconds, serviceIdentifier = serviceIdentifier, statsReceiver = statsReceiver ) buildMemcacheClient(cacheClient, "/srv#/prod/local/cache/tv_real_time_aggregates:twemcaches") } @Provides @Singleton @Named(EntityRealGraphClientStore) def providesEntityRealGraphClient( serviceIdentifier: ServiceIdentifier ): ReadableStore[erg.EntityRealGraphRequest, erg.EntityRealGraphResponse] = { EntityRealGraphClient(Environment.withName("prod"), serviceIdentifier, Some(250.millis)) } private def buildMemcacheClient(cacheClient: Memcached.Client, dest: String): Memcache = FinagleMemcacheFactory( client = cacheClient, dest = dest, hashName = memcacheKeyHasher )() } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/MemcachedScoredCandidateFeaturesStoreModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.storehaus.Store import com.twitter.timelines.clients.memcache_common.StorehausMemcacheConfig import com.twitter.timelines.ml.cont_train.common.client.scored_candidate_features_cache.ScoredCandidateFeaturesMemcacheBuilder import com.twitter.timelines.served_candidates_logging.{thriftscala => scl} import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pdr} import com.twitter.home_mixer.param.HomeMixerInjectionNames.MemcacheCandidateFeaturesStore import com.twitter.home_mixer.param.HomeMixerInjectionNames.MemcacheVideoCandidateFeaturesStore import com.twitter.finagle.memcached.compressing.scheme.Lz4 import javax.inject.Named import javax.inject.Singleton object MemcachedScoredCandidateFeaturesStoreModule extends TwitterModule { private val ScoredTweetsProdDestName = "/srv#/prod/local/cache/timelines_scored_candidate_features:twemcaches" private val ScoredTweetsStagingDestName = "/srv#/test/local/cache/twemcache_timelines_scored_candidate_features:twemcaches" private val ScoredVideoProdDestName = "/srv#/prod/local/cache/timelines_scored_video_candidate_features:twemcaches" private val ScoredVideoStagingDestName = "/srv#/test/local/cache/twemcache_timelines_scored_video_candidate_features:twemcaches" @Singleton @Provides @Named(MemcacheCandidateFeaturesStore) def providesMemcachedScoredCandidateFeaturesStore( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Store[scl.CandidateFeatureKey, pdr.PolyDataRecord] = { val destName = serviceIdentifier.environment.toLowerCase match { case "prod" => ScoredTweetsProdDestName case _ => ScoredTweetsStagingDestName } buildCacheClient(serviceIdentifier, statsReceiver, destName) } @Singleton @Provides @Named(MemcacheVideoCandidateFeaturesStore) def providesMemcachedScoredVideoCandidateFeaturesStore( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Store[scl.CandidateFeatureKey, pdr.PolyDataRecord] = { val destName = serviceIdentifier.environment.toLowerCase match { case "prod" => ScoredVideoProdDestName case _ => ScoredVideoStagingDestName } buildCacheClient(serviceIdentifier, statsReceiver, destName) } private def buildCacheClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver, destName: String, ): Store[scl.CandidateFeatureKey, pdr.PolyDataRecord] = { new ScoredCandidateFeaturesMemcacheBuilder( config = StorehausMemcacheConfig( destName = destName, keyPrefix = "", requestTimeout = 200.milliseconds, numTries = 2, globalTimeout = 500.milliseconds, tcpConnectTimeout = 20.milliseconds, connectionAcquisitionTimeout = 150.milliseconds, numPendingRequests = 200, isReadOnly = false, serviceIdentifier = serviceIdentifier, numConnections = 1, compressionScheme = Lz4 ), ttl = 5.minute, statsReceiver = statsReceiver ).build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/NaviModelClientModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.Http import com.twitter.finagle.grpc.FinagleChannelBuilder import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsStackClientSyntax import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecap import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapGPU import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapRealtime import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapSecondary import com.twitter.home_mixer.param.HomeMixerInjectionNames.NaviModelClientHomeRecapVideo import com.twitter.inject.TwitterModule import com.twitter.timelines.clients.predictionservice.PredictionGRPCService import com.twitter.util.Duration import io.grpc.ManagedChannel import javax.inject.Named import javax.inject.Singleton object NaviModelClientModule extends TwitterModule { private val Authority = "rustserving" private val MaxRetryAttempts = 2 private val MaxPredictionTimeoutMs: Duration = 700.millis private val ConnectTimeoutMs: Duration = 200.millis private val AcquisitionTimeoutMs: Duration = 500.millis @Singleton @Named(NaviModelClientHomeRecap) @Provides def providesHomeRecapPredictionGRPCService( serviceIdentifier: ServiceIdentifier, ): PredictionGRPCService = { providesPredictionGRPCService(serviceIdentifier, "navi_home_recap_onnx") } @Singleton @Named(NaviModelClientHomeRecapSecondary) @Provides def providesHomeRecapSecondaryPredictionGRPCService( serviceIdentifier: ServiceIdentifier, ): PredictionGRPCService = { providesPredictionGRPCService(serviceIdentifier, "navi_home_recap_onnx_2") } @Singleton @Named(NaviModelClientHomeRecapRealtime) @Provides def providesHomeRecapRealtimePredictionGRPCService( serviceIdentifier: ServiceIdentifier, ): PredictionGRPCService = { providesPredictionGRPCService(serviceIdentifier, "navi_home_realtime_recap_onnx") } @Singleton @Named(NaviModelClientHomeRecapGPU) @Provides def providesHomeRecapGPUPredictionGRPCService( serviceIdentifier: ServiceIdentifier, ): PredictionGRPCService = { providesPredictionGRPCService(serviceIdentifier, "navi_home_recap_onnx_v100") } @Singleton @Named(NaviModelClientHomeRecapVideo) @Provides def providesHomeRecapVideoPredictionGRPCService( serviceIdentifier: ServiceIdentifier, ): PredictionGRPCService = { providesPredictionGRPCService(serviceIdentifier, "navi_home_recap_video_onnx") } private def providesPredictionGRPCService( serviceIdentifier: ServiceIdentifier, naviClusterName: String ): PredictionGRPCService = { val modelPath = s"/s/ml-serving/$naviClusterName" val client = Http.client .withLabel(modelPath) .withMutualTls(serviceIdentifier) .withRequestTimeout(MaxPredictionTimeoutMs) .withTransport.connectTimeout(ConnectTimeoutMs) .withSession.acquisitionTimeout(AcquisitionTimeoutMs) .withHttpStats val channel: ManagedChannel = FinagleChannelBuilder .forTarget(modelPath) .overrideAuthority(Authority) .maxRetryAttempts(MaxRetryAttempts) .enableRetryForStatus(io.grpc.Status.RESOURCE_EXHAUSTED) .enableRetryForStatus(io.grpc.Status.UNKNOWN) .enableUnsafeFullyBufferingMode() .httpClient(client) .build() new PredictionGRPCService(channel) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/OptimizedStratoClientModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.service.Retries import com.twitter.finagle.service.RetryPolicy import com.twitter.finagle.ssl.OpportunisticTls import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithDefaultTimeout import com.twitter.home_mixer.param.HomeMixerInjectionNames.StratoClientWithLongTimeout import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithLongTimeout import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout import com.twitter.home_mixer.param.HomeMixerInjectionNames.TesBatchedStratoClient import com.twitter.inject.TwitterModule import com.twitter.strato.client.Client import com.twitter.strato.client.Strato import com.twitter.strato.rpc.ClientBucketingStrategy import com.twitter.util.Try import javax.inject.Named import javax.inject.Singleton /** * Strato Finagle config is copied from TLX */ object OptimizedStratoClientModule extends TwitterModule { private val StratoClientConnectionTimeout = 200.millis private val StratoClientAcquisitionTimeout = 500.millis private val DefaultStratoClientRequestTimeout = 280.millis private val StratoClientLongRequestTimeout = 1000.millis private val ModerateStratoClientRequestTimeout = 600.millis private val LongStratoClientRequestTimeout = 1000.millis private val DefaultRetryPartialFunction: PartialFunction[Try[Nothing], Boolean] = RetryPolicy.TimeoutAndWriteExceptionsOnly .orElse(RetryPolicy.ChannelClosedExceptionsOnly) protected def mkRetryPolicy(tries: Int): RetryPolicy[Try[Nothing]] = RetryPolicy.tries(tries, DefaultRetryPartialFunction) @Singleton @Provides @Named(BatchedStratoClientWithDefaultTimeout) def providesDefaultStratoClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Client = { Strato.client .withMutualTls(serviceIdentifier, opportunisticLevel = OpportunisticTls.Required) .withSession.acquisitionTimeout(StratoClientAcquisitionTimeout) .withTransport.connectTimeout(StratoClientConnectionTimeout) .withRequestTimeout(DefaultStratoClientRequestTimeout) .withPerRequestTimeout(DefaultStratoClientRequestTimeout) .configured(Retries.Policy(mkRetryPolicy(1))) .withStatsReceiver(statsReceiver.scope("default_strato_client")) .build() } @Singleton @Provides @Named(StratoClientWithLongTimeout) def providesLongStratoClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Client = { Strato.client .withMutualTls(serviceIdentifier, opportunisticLevel = OpportunisticTls.Required) .withSession.acquisitionTimeout(StratoClientAcquisitionTimeout) .withTransport.connectTimeout(StratoClientConnectionTimeout) .withRequestTimeout(StratoClientLongRequestTimeout) .withPerRequestTimeout(StratoClientLongRequestTimeout) .configured(Retries.Policy(mkRetryPolicy(10))) .withStatsReceiver(statsReceiver.scope("long_strato_client")) .build() } @Singleton @Provides @Named(BatchedStratoClientWithModerateTimeout) def providesModerateTimeoutStratoClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Client = { Strato.client .withMutualTls(serviceIdentifier, opportunisticLevel = OpportunisticTls.Required) .withSession.acquisitionTimeout(StratoClientAcquisitionTimeout) .withTransport.connectTimeout(StratoClientConnectionTimeout) .withRequestTimeout(ModerateStratoClientRequestTimeout) .withPerRequestTimeout(ModerateStratoClientRequestTimeout) .withRpcBatchSize(64) .configured(Retries.Policy(mkRetryPolicy(1))) .withStatsReceiver(statsReceiver.scope("moderate_timeout_strato_client")) .build() } @Singleton @Provides @Named(BatchedStratoClientWithLongTimeout) def providesLongTimeoutStratoClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Client = { Strato.client .withMutualTls(serviceIdentifier, opportunisticLevel = OpportunisticTls.Required) .withSession.acquisitionTimeout(StratoClientAcquisitionTimeout) .withTransport.connectTimeout(StratoClientConnectionTimeout) .withRequestTimeout(LongStratoClientRequestTimeout) .withPerRequestTimeout(LongStratoClientRequestTimeout) .withRpcBatchSize(5) .configured(Retries.Policy(mkRetryPolicy(1))) .withStatsReceiver(statsReceiver.scope("long_timeout_strato_client")) .build() } @Singleton @Provides @Named(TesBatchedStratoClient) def providesTesStratoClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): Client = { Strato.client .withMutualTls(serviceIdentifier, opportunisticLevel = OpportunisticTls.Required) .withSession.acquisitionTimeout(StratoClientAcquisitionTimeout) .withTransport.connectTimeout(StratoClientConnectionTimeout) .withRequestTimeout(ModerateStratoClientRequestTimeout) .withPerRequestTimeout(ModerateStratoClientRequestTimeout) .withRpcBatchSize(140) .withBucketingStrategy(ClientBucketingStrategy.ByArg) .configured(Retries.Policy(mkRetryPolicy(1))) .withStatsReceiver(statsReceiver.scope("tes_strato_client")) .build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PeopleDiscoveryServiceModule.scala ================================================ package com.twitter.home_mixer.module import com.twitter.conversions.DurationOps._ import com.twitter.finagle.thriftmux.MethodBuilder import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.inject.Injector import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.peoplediscovery.api.thriftscala.ThriftPeopleDiscoveryService import com.twitter.util.Duration /** * Copy of com.twitter.product_mixer.component_library.module.PeopleDiscoveryServiceModule */ object PeopleDiscoveryServiceModule extends ThriftMethodBuilderClientModule[ ThriftPeopleDiscoveryService.ServicePerEndpoint, ThriftPeopleDiscoveryService.MethodPerEndpoint ] with MtlsClient { override val label: String = "people-discovery-api" override val dest: String = "/s/people-discovery-api/people-discovery-api:thrift" override protected def configureMethodBuilder( injector: Injector, methodBuilder: MethodBuilder ): MethodBuilder = { methodBuilder .withTimeoutPerRequest(350.millis) .withTimeoutTotal(350.millis) } override protected def sessionAcquisitionTimeout: Duration = 500.milliseconds } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PhoenixClientModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.google.inject.name.Named import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Experiment1 import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Experiment2 import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Experiment3 import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Experiment4 import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Experiment5 import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Experiment6 import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Experiment7 import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Experiment8 import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster.Prod import com.twitter.inject.TwitterModule import io.grpc.ManagedChannel import io.grpc.netty.NettyChannelBuilder import java.util.concurrent.TimeUnit import javax.inject.Singleton object PhoenixClientModule extends TwitterModule { val ChannelsPerHost = 10 val XAIGrpcPort = 80 val ProdEndpoint = "" private def buildChannels(endpoint: String): Seq[ManagedChannel] = { val endpoints = Seq.fill(ChannelsPerHost)(endpoint) endpoints.map { host => NettyChannelBuilder .forAddress(host, XAIGrpcPort) .usePlaintext() .keepAliveTime(60, TimeUnit.SECONDS) .keepAliveTimeout(20, TimeUnit.SECONDS) .keepAliveWithoutCalls(true) .initialFlowControlWindow(128 * 1024 * 1024) .flowControlWindow(1024 * 1024 * 128) .maxInboundMessageSize(20 * 1024 * 1024) .build() } } @Provides @Singleton @Named("PhoenixClient") private def getStub(): Map[PhoenixCluster.Value, Seq[ManagedChannel]] = { val endpointMap = Map( Prod -> ProdEndpoint, Experiment1 -> "", Experiment2 -> "", Experiment3 -> "", Experiment4 -> "", Experiment5 -> "", Experiment6 -> "", Experiment7 -> "", Experiment8 -> "", ).withDefaultValue(ProdEndpoint) endpointMap.mapValues(buildChannels) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/PipelineFailureExceptionMapper.scala ================================================ package com.twitter.home_mixer.module import com.twitter.finatra.thrift.exceptions.ExceptionMapper import com.twitter.home_mixer.{thriftscala => t} import com.twitter.util.logging.Logging import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.product_mixer.core.pipeline.pipeline_failure.ProductDisabled import com.twitter.scrooge.ThriftException import com.twitter.util.Future import javax.inject.Singleton @Singleton class PipelineFailureExceptionMapper extends ExceptionMapper[PipelineFailure, ThriftException] with Logging { def handleException(throwable: PipelineFailure): Future[ThriftException] = { throwable match { // SliceService (unlike UrtService) throws an exception when the requested product is disabled case PipelineFailure(ProductDisabled, reason, _, _) => Future.exception( t.ValidationExceptionList(errors = Seq(t.ValidationException(t.ValidationErrorCode.ProductDisabled, reason)))) case _ => error("Unhandled PipelineFailure", throwable) Future.exception(throwable) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealGraphInNetworkScoresModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.google.inject.name.Named import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphInNetworkScoresOnPrem import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphManhattanEndpoint import com.twitter.home_mixer.store.RealGraphInNetworkScoresStore import com.twitter.inject.TwitterModule import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint import com.twitter.storehaus.ReadableStore import com.twitter.timelines.util.CommonTypes.ViewerId import com.twitter.wtf.candidate.thriftscala.Candidate import javax.inject.Singleton object RealGraphInNetworkScoresModule extends TwitterModule { @Provides @Singleton @Named(RealGraphInNetworkScoresOnPrem) def providesRealGraphInNetworkScoresFeaturesStore( @Named(RealGraphManhattanEndpoint) realGraphInNetworkScoresManhattanKVEndpoint: ManhattanKVEndpoint ): ReadableStore[ViewerId, Seq[Candidate]] = { new RealGraphInNetworkScoresStore(realGraphInNetworkScoresManhattanKVEndpoint) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/RealtimeAggregateFeatureRepositoryModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.google.inject.name.Named import com.twitter.bijection.Injection import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.bijection.thrift.ThriftCodec import com.twitter.home_mixer.param.HomeMixerInjectionNames.EngagementsReceivedByAuthorCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionGraphUserVertexCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionGraphUserVertexClient import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelinesRealTimeAggregateClient import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicCountryEngagementCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicEngagementCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.TvRealTimeAggregateClient import com.twitter.home_mixer.param.HomeMixerInjectionNames.TvVideoByUserTweetCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetCountryEngagementCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetEngagementCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwitterListEngagementCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserAuthorEngagementCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserEngagementCache import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserTopicEngagementForNewUserCache import com.twitter.home_mixer.util.InjectionTransformerImplicits._ import com.twitter.inject.TwitterModule import com.twitter.ml.api.DataRecord import com.twitter.ml.api.Feature import com.twitter.ml.{api => ml} import com.twitter.servo.cache.KeyValueTransformingReadCache import com.twitter.servo.cache.Memcache import com.twitter.servo.cache.ReadCache import com.twitter.servo.util.Transformer import com.twitter.storehaus_internal.memcache.MemcacheHelper import com.twitter.summingbird.batch.Batcher import com.twitter.summingbird_internal.bijection.BatchPairImplicits import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregationKey import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregationKeyInjection import com.twitter.wtf.real_time_interaction_graph.{thriftscala => ig} import javax.inject.Singleton object RealtimeAggregateFeatureRepositoryModule extends TwitterModule with RealtimeAggregateHelpers { @Provides @Singleton @Named(UserTopicEngagementForNewUserCache) def providesUserTopicEngagementForNewUserCache( @Named(TimelinesRealTimeAggregateClient) client: Memcache ): ReadCache[(Long, Long), ml.DataRecord] = { new KeyValueTransformingReadCache( client, dataRecordValueTransformer, keyTransformD2(userIdFeature, topicIdFeature) ) } @Provides @Singleton @Named(TwitterListEngagementCache) def providesTwitterListEngagementCache( @Named(TimelinesRealTimeAggregateClient) client: Memcache ): ReadCache[Long, ml.DataRecord] = { new KeyValueTransformingReadCache( client, dataRecordValueTransformer, keyTransformD1(listIdFeature) ) } @Provides @Singleton @Named(TopicEngagementCache) def providesTopicEngagementCache( @Named(TimelinesRealTimeAggregateClient) client: Memcache ): ReadCache[Long, ml.DataRecord] = { new KeyValueTransformingReadCache( client, dataRecordValueTransformer, keyTransformD1(topicIdFeature) ) } @Provides @Singleton @Named(UserAuthorEngagementCache) def providesUserAuthorEngagementCache( @Named(TimelinesRealTimeAggregateClient) client: Memcache ): ReadCache[(Long, Long), ml.DataRecord] = { new KeyValueTransformingReadCache( client, dataRecordValueTransformer, keyTransformD2(userIdFeature, authorIdFeature) ) } @Provides @Singleton @Named(UserEngagementCache) def providesUserEngagementCache( @Named(TimelinesRealTimeAggregateClient) client: Memcache ): ReadCache[Long, ml.DataRecord] = { new KeyValueTransformingReadCache( client, dataRecordValueTransformer, keyTransformD1(userIdFeature) ) } @Provides @Singleton @Named(TweetCountryEngagementCache) def providesTweetCountryEngagementCache( @Named(TimelinesRealTimeAggregateClient) client: Memcache ): ReadCache[(Long, String), ml.DataRecord] = { new KeyValueTransformingReadCache( client, dataRecordValueTransformer, keyTransformD1T1(tweetIdFeature, countryCodeFeature) ) } @Provides @Singleton @Named(TweetEngagementCache) def providesTweetEngagementCache( @Named(TimelinesRealTimeAggregateClient) client: Memcache ): ReadCache[Long, ml.DataRecord] = { new KeyValueTransformingReadCache( client, dataRecordValueTransformer, keyTransformD1(tweetIdFeature) ) } @Provides @Singleton @Named(EngagementsReceivedByAuthorCache) def providesEngagementsReceivedByAuthorCache( @Named(TimelinesRealTimeAggregateClient) client: Memcache ): ReadCache[Long, ml.DataRecord] = { new KeyValueTransformingReadCache( client, dataRecordValueTransformer, keyTransformD1(authorIdFeature) ) } @Provides @Singleton @Named(TopicCountryEngagementCache) def providesTopicCountryEngagementCache( @Named(TimelinesRealTimeAggregateClient) client: Memcache ): ReadCache[(Long, String), ml.DataRecord] = { new KeyValueTransformingReadCache( client, dataRecordValueTransformer, keyTransformD1T1(topicIdFeature, countryCodeFeature) ) } @Provides @Singleton @Named(RealTimeInteractionGraphUserVertexCache) def providesRealTimeInteractionGraphUserVertexCache( @Named(RealTimeInteractionGraphUserVertexClient) client: Memcache ): ReadCache[Long, ig.UserVertex] = { val valueTransformer = BinaryScalaCodec(ig.UserVertex).toByteArrayTransformer() val underlyingKey: Long => String = { val cacheKeyPrefix = "user_vertex" val defaultBatchID = Batcher.unit.currentBatch val batchPairInjection = BatchPairImplicits.keyInjection(Injection.connect[Long, Array[Byte]]) MemcacheHelper .keyEncoder(cacheKeyPrefix)(batchPairInjection) .compose((k: Long) => (k, defaultBatchID)) } new KeyValueTransformingReadCache( client, valueTransformer, underlyingKey ) } @Provides @Singleton @Named(TvVideoByUserTweetCache) def providesTvVideoImpressionByUserTweetCache( @Named(TvRealTimeAggregateClient) client: Memcache ): ReadCache[(Long, Long), ml.DataRecord] = { new KeyValueTransformingReadCache( client, dataRecordValueTransformer, keyTransformD2(userIdFeature, tweetIdFeature) ) } } trait RealtimeAggregateHelpers { val authorIdFeature = new Feature.Discrete("entities.source_author_id").getFeatureId val countryCodeFeature = new Feature.Text("geo.user_location.country_code").getFeatureId val listIdFeature = new Feature.Discrete("list.id").getFeatureId val userIdFeature = new Feature.Discrete("meta.user_id").getFeatureId val topicIdFeature = new Feature.Discrete("entities.topic_id").getFeatureId val tweetIdFeature = new Feature.Discrete("entities.source_tweet_id").getFeatureId private def customKeyBuilder[K](prefix: String, f: K => Array[Byte]): K => String = { // intentionally not implementing injection inverse because it is never used def g(arr: Array[Byte]) = ??? MemcacheHelper.keyEncoder(prefix)(Injection.build(f)(g)) } private val keyEncoder: AggregationKey => String = { val cacheKeyPrefix = "" val defaultBatchID = Batcher.unit.currentBatch val batchPairInjection = BatchPairImplicits.keyInjection(AggregationKeyInjection) customKeyBuilder(cacheKeyPrefix, batchPairInjection) .compose((k: AggregationKey) => (k, defaultBatchID)) } def keyTransformD1AggregationKey(f1: Long)(key: Long): AggregationKey = { AggregationKey(Map(f1 -> key), Map.empty) } def keyTransformD1(f1: Long)(key: Long): String = { val aggregationKey = AggregationKey(Map(f1 -> key), Map.empty) keyEncoder(aggregationKey) } def keyTransformD2(f1: Long, f2: Long)(keys: (Long, Long)): String = { val (k1, k2) = keys val aggregationKey = AggregationKey(Map(f1 -> k1, f2 -> k2), Map.empty) keyEncoder(aggregationKey) } def keyTransformD2AggregationKey(f1: Long, f2: Long)(keys: (Long, Long)): AggregationKey = { val (k1, k2) = keys AggregationKey(Map(f1 -> k1, f2 -> k2), Map.empty) } def keyTransformD1T1(f1: Long, f2: Long)(keys: (Long, String)): String = { val (k1, k2) = keys val aggregationKey = AggregationKey(Map(f1 -> k1), Map(f2 -> k2)) keyEncoder(aggregationKey) } def keyTransformD1T1AggregationKey(f1: Long, f2: Long)(keys: (Long, String)): AggregationKey = { val (k1, k2) = keys AggregationKey(Map(f1 -> k1), Map(f2 -> k2)) } val dataRecordValueTransformer: Transformer[DataRecord, Array[Byte]] = ThriftCodec .toCompact[ml.DataRecord] .toByteArrayTransformer() } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredTweetsMemcacheModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.ScoredTweetsCache import com.twitter.home_mixer.{thriftscala => t} import com.twitter.inject.TwitterModule import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder import com.twitter.servo.cache.FinagleMemcache import com.twitter.servo.cache.KeyTransformer import com.twitter.servo.cache.KeyValueTransformingTtlCache import com.twitter.servo.cache.Serializer import com.twitter.servo.cache.ThriftSerializer import com.twitter.servo.cache.TtlCache import com.twitter.timelines.model.UserId import org.apache.thrift.protocol.TCompactProtocol import com.twitter.finagle.memcached.compressing.scheme.Lz4 import javax.inject.Named import javax.inject.Singleton object ScoredTweetsMemcacheModule extends TwitterModule { private val ScopeName = "ScoredTweetsCache" private val ProdDestName = "/srv#/prod/local/cache/home_scored_tweets:twemcaches" private val StagingDestName = "/srv#/test/local/cache/twemcache_home_scored_tweets:twemcaches" private val scoredTweetsSerializer: Serializer[t.ScoredTweetsResponse] = new ThriftSerializer[t.ScoredTweetsResponse]( t.ScoredTweetsResponse, new TCompactProtocol.Factory()) private val userIdKeyTransformer: KeyTransformer[UserId] = (userId: UserId) => userId.toString @Singleton @Named(ScoredTweetsCache) @Provides def providesScoredTweetsCache( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): TtlCache[UserId, t.ScoredTweetsResponse] = { val destName = serviceIdentifier.environment.toLowerCase match { case "prod" => ProdDestName case _ => StagingDestName } val client = MemcachedClientBuilder.buildMemcachedClient( destName = destName, numTries = 2, numConnections = 1, requestTimeout = 200.milliseconds, globalTimeout = 400.milliseconds, connectTimeout = 100.milliseconds, acquisitionTimeout = 100.milliseconds, serviceIdentifier = serviceIdentifier, statsReceiver = statsReceiver.scope(ScopeName), compressionScheme = Lz4 ) val underlyingCache = new FinagleMemcache(client) new KeyValueTransformingTtlCache( underlyingCache = underlyingCache, transformer = scoredTweetsSerializer, underlyingKey = userIdKeyTransformer ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScoredVideoTweetsMemcacheModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.memcached.compressing.scheme.Lz4 import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.ScoredVideoTweetsCache import com.twitter.home_mixer.{thriftscala => t} import com.twitter.inject.TwitterModule import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder import com.twitter.servo.cache._ import com.twitter.timelines.model.UserId import org.apache.thrift.protocol.TCompactProtocol import javax.inject.Named import javax.inject.Singleton object ScoredVideoTweetsMemcacheModule extends TwitterModule { private val ScopeName = "ScoredVideoTweetsCache" private val destName = "/s/cache/explore_served_tweet_impressions:twemcaches" private val scoredTweetsSerializer: Serializer[t.ScoredTweetsResponse] = new ThriftSerializer[t.ScoredTweetsResponse]( t.ScoredTweetsResponse, new TCompactProtocol.Factory()) private val userIdKeyTransformer: KeyTransformer[UserId] = (userId: UserId) => userId.toString + ":home-mixer" @Singleton @Named(ScoredVideoTweetsCache) @Provides def providesScoredTweetsCache( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): TtlCache[UserId, t.ScoredTweetsResponse] = { val client = MemcachedClientBuilder.buildMemcachedClient( destName = destName, numTries = 2, numConnections = 1, requestTimeout = 200.milliseconds, globalTimeout = 400.milliseconds, connectTimeout = 100.milliseconds, acquisitionTimeout = 100.milliseconds, serviceIdentifier = serviceIdentifier, statsReceiver = statsReceiver.scope(ScopeName), compressionScheme = Lz4 ) val underlyingCache = new FinagleMemcache(client) new KeyValueTransformingTtlCache( underlyingCache = underlyingCache, transformer = scoredTweetsSerializer, underlyingKey = userIdKeyTransformer ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ScribeEventPublisherModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.clientapp.{thriftscala => ca} import com.twitter.finatra.kafka.interceptors.InstanceMetadataProducerInterceptor import com.twitter.finatra.kafka.interceptors.PublishTimeProducerInterceptor import com.twitter.finatra.kafka.producers.FinagleKafkaProducerBuilder import com.twitter.finatra.kafka.serde.ScalaSerdes import com.twitter.finatra.kafka.serde.UnKeyedSerde import com.twitter.home_mixer.param.HomeMixerInjectionNames.CommonFeaturesScribeEventPublisher import com.twitter.home_mixer.param.HomeMixerInjectionNames.CommonFeaturesScribeVideoEventPublisher import com.twitter.inject.TwitterModule import com.twitter.logpipeline.client.EventPublisherManager import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.logpipeline.client.serializers.EventLogMsgTBinarySerializer import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} import com.twitter.timelines.timeline_logging.{thriftscala => tl} import com.twitter.util.Duration import com.twitter.util.StorageUnit import javax.inject.Named import javax.inject.Singleton import org.apache.kafka.clients.CommonClientConfigs import org.apache.kafka.common.config.SaslConfigs import org.apache.kafka.common.config.SslConfigs import org.apache.kafka.common.security.auth.SecurityProtocol object ScribeEventPublisherModule extends TwitterModule { val ClientEventLogCategory = "client_event" val ServedCandidatesLogCategory = "home_timeline_served_candidates_flattened" val ScoredCandidatesLogCategory = "home_timeline_scored_candidates" val ServedCommonFeaturesLogCategory = "tq_served_common_features_offline" val ServedVideoCommonFeaturesLogCategory = "tq_served_video_common_features_offline" @Provides @Singleton def providesClientEventsScribeEventPublisher: EventPublisher[ca.LogEvent] = { val builder = FinagleKafkaProducerBuilder() .dest(s"/s/kafka/client-events:kafka-tls") .keySerializer(UnKeyedSerde.serializer) .valueSerializer(ScalaSerdes.Thrift[ca.LogEvent].serializer) .clientId("home_mixer_client_event_publisher") .linger(Duration.fromMilliseconds(16)) .batchSize(StorageUnit.fromKilobytes(64)) .deliveryTimeout(Duration.fromMilliseconds(30000)) .requestTimeout(Duration.fromMilliseconds(25000)) .withConfig(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_SSL.toString) .interceptor[PublishTimeProducerInterceptor] .interceptor[InstanceMetadataProducerInterceptor] .withConfig(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "") .withConfig(SaslConfigs.SASL_MECHANISM, SaslConfigs.GSSAPI_MECHANISM) .withConfig(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "kafka") .withConfig(SaslConfigs.SASL_KERBEROS_SERVER_NAME, "kafka") EventPublisherManager.buildKafkaLogPipelinePublisher(builder, ClientEventLogCategory) } @Provides @Singleton @Named(CommonFeaturesScribeEventPublisher) def providesCommonFeaturesScribeEventPublisher: EventPublisher[pldr.PolyDataRecord] = { val serializer = EventLogMsgTBinarySerializer.getNewSerializer EventPublisherManager.buildScribeLogPipelinePublisher( ServedCommonFeaturesLogCategory, serializer) } @Provides @Singleton @Named(CommonFeaturesScribeVideoEventPublisher) def providesCommonFeaturesScribeVideoEventPublisher: EventPublisher[pldr.PolyDataRecord] = { val serializer = EventLogMsgTBinarySerializer.getNewSerializer EventPublisherManager.buildScribeLogPipelinePublisher( ServedVideoCommonFeaturesLogCategory, serializer) } @Provides @Singleton def providesServedCandidatesScribeEventPublisher: EventPublisher[tl.ServedEntry] = { val builder = FinagleKafkaProducerBuilder() .dest("/s/kafka/timeline:kafka-tls") .keySerializer(UnKeyedSerde.serializer) .valueSerializer(ScalaSerdes.Thrift[tl.ServedEntry].serializer) .clientId(s"$ServedCandidatesLogCategory-publisher") .linger(Duration.fromMilliseconds(16)) .batchSize(StorageUnit.fromKilobytes(64)) .deliveryTimeout(Duration.fromMilliseconds(30000)) .requestTimeout(Duration.fromMilliseconds(25000)) .withConfig(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_SSL.toString) .interceptor[PublishTimeProducerInterceptor] .interceptor[InstanceMetadataProducerInterceptor] .withConfig(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "") .withConfig(SaslConfigs.SASL_MECHANISM, SaslConfigs.GSSAPI_MECHANISM) .withConfig(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "kafka") .withConfig(SaslConfigs.SASL_KERBEROS_SERVER_NAME, "kafka") EventPublisherManager.buildKafkaLogPipelinePublisher(builder, ServedCandidatesLogCategory) } @Provides @Singleton def provideScoredCandidatesScribeEventPublisher: EventPublisher[tl.ScoredCandidate] = { val builder = FinagleKafkaProducerBuilder() .dest("/s/kafka/timeline:kafka-tls") .keySerializer(UnKeyedSerde.serializer) .valueSerializer(ScalaSerdes.Thrift[tl.ScoredCandidate].serializer) .clientId(s"$ScoredCandidatesLogCategory-publisher") .linger(Duration.fromMilliseconds(16)) .batchSize(StorageUnit.fromKilobytes(64)) .deliveryTimeout(Duration.fromMilliseconds(30000)) .requestTimeout(Duration.fromMilliseconds(25000)) .withConfig(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_SSL.toString) .interceptor[PublishTimeProducerInterceptor] .interceptor[InstanceMetadataProducerInterceptor] .withConfig(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "") .withConfig(SaslConfigs.SASL_MECHANISM, SaslConfigs.GSSAPI_MECHANISM) .withConfig(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "kafka") .withConfig(SaslConfigs.SASL_KERBEROS_SERVER_NAME, "kafka") EventPublisherManager.buildKafkaLogPipelinePublisher(builder, ScoredCandidatesLogCategory) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/SimClustersRecentEngagementsClientModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout import com.twitter.inject.TwitterModule import com.twitter.strato.client.Client import com.twitter.timelines.clients.strato.twistly.SimClustersRecentEngagementSimilarityClient import com.twitter.timelines.clients.strato.twistly.SimClustersRecentEngagementSimilarityClientImpl import javax.inject.Named import javax.inject.Singleton object SimClustersRecentEngagementsClientModule extends TwitterModule { @Singleton @Provides def providesSimilarityClient( @Named(BatchedStratoClientWithModerateTimeout) stratoClient: Client, statsReceiver: StatsReceiver ): SimClustersRecentEngagementSimilarityClient = { new SimClustersRecentEngagementSimilarityClientImpl(stratoClient, statsReceiver) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/StaleTweetsCacheModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.google.inject.name.Named import com.twitter.conversions.DurationOps._ import com.twitter.finagle.memcached.{Client => MemcachedClient} import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.hashing.KeyHasher import com.twitter.home_mixer.param.HomeMixerInjectionNames.StaleTweetsCache import com.twitter.inject.TwitterModule import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder import javax.inject.Singleton object StaleTweetsCacheModule extends TwitterModule { @Singleton @Provides @Named(StaleTweetsCache) def providesCache( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): MemcachedClient = { MemcachedClientBuilder.buildMemcachedClient( destName = "/srv#/prod/local/cache/staletweetscache:twemcaches", numTries = 3, numConnections = 1, requestTimeout = 200.milliseconds, globalTimeout = 500.milliseconds, connectTimeout = 200.milliseconds, acquisitionTimeout = 200.milliseconds, serviceIdentifier = serviceIdentifier, statsReceiver = statsReceiver, failureAccrualPolicy = None, keyHasher = Some(KeyHasher.FNV1_32) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/ThriftFeatureRepositoryModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.conversions.PercentOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.thrift.ClientId import com.twitter.graph_feature_service.{thriftscala => gfs} import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.GraphTwoHopRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.InterestsThriftServiceClient import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserFollowedTopicIdsRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.UtegSocialProofRepository import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil import com.twitter.inject.TwitterModule import com.twitter.interests.{thriftscala => int} import com.twitter.product_mixer.shared_library.thrift_client.FinagleThriftClientBuilder import com.twitter.product_mixer.shared_library.thrift_client.Idempotent import com.twitter.recos.recos_common.{thriftscala => rc} import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.servo.cache._ import com.twitter.servo.keyvalue.KeyValueResultBuilder import com.twitter.servo.repository.ChunkingStrategy import com.twitter.servo.repository.KeyValueRepository import com.twitter.servo.repository.KeyValueResult import com.twitter.util.Future import com.twitter.util.Return import javax.inject.Named import javax.inject.Singleton object ThriftFeatureRepositoryModule extends TwitterModule { private val DefaultRPCChunkSize = 50 private val GFSInteractionIdsLimit = 10 type EarlybirdQuery = (Seq[Long], Long) type UtegQuery = (Seq[Long], (Long, Map[Long, Double])) @Provides @Singleton @Named(InterestsThriftServiceClient) def providesInterestsThriftServiceClient( clientId: ClientId, serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): int.InterestsThriftService.MethodPerEndpoint = { FinagleThriftClientBuilder .buildFinagleMethodPerEndpoint[ int.InterestsThriftService.ServicePerEndpoint, int.InterestsThriftService.MethodPerEndpoint]( serviceIdentifier = serviceIdentifier, clientId = clientId, dest = "/s/interests-thrift-service/interests-thrift-service", label = "interests", statsReceiver = statsReceiver, idempotency = Idempotent(1.percent), timeoutPerRequest = 350.milliseconds, timeoutTotal = 350.milliseconds ) } @Provides @Singleton @Named(UserFollowedTopicIdsRepository) def providesUserFollowedTopicIdsRepository( @Named(InterestsThriftServiceClient) client: int.InterestsThriftService.MethodPerEndpoint ): KeyValueRepository[Seq[Long], Long, Seq[Long]] = { val lookupContext = Some( int.ExplicitInterestLookupContext(Some(Seq(int.InterestRelationType.Followed))) ) def lookup(userId: Long): Future[Seq[Long]] = { client.getUserExplicitInterests(userId, lookupContext).map { interests => interests.flatMap { _.interestId match { case int.InterestId.SemanticCore(semanticCoreInterest) => Some(semanticCoreInterest.id) case _ => None } } } } val keyValueRepository = toRepository(lookup) keyValueRepository } @Provides @Singleton @Named(UtegSocialProofRepository) def providesUtegSocialProofRepository( clientId: ClientId, serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): KeyValueRepository[UtegQuery, Long, uteg.TweetRecommendation] = { val client = FinagleThriftClientBuilder.buildFinagleMethodPerEndpoint[ uteg.UserTweetEntityGraph.ServicePerEndpoint, uteg.UserTweetEntityGraph.MethodPerEndpoint]( serviceIdentifier = serviceIdentifier, clientId = clientId, dest = "/s/cassowary/user_tweet_entity_graph", label = "uteg-social-proof-repo", statsReceiver = statsReceiver, idempotency = Idempotent(1.percent), timeoutPerRequest = 150.milliseconds, timeoutTotal = 250.milliseconds ) val utegSocialProofTypes = Seq( rc.SocialProofType.Favorite, rc.SocialProofType.Retweet, rc.SocialProofType.Reply ) def lookup( tweetIds: Seq[Long], view: (Long, Map[Long, Double]) ): Future[Seq[Option[uteg.TweetRecommendation]]] = { val (userId, seedsWithWeights) = view val socialProofRequest = uteg.SocialProofRequest( requesterId = Some(userId), seedsWithWeights = seedsWithWeights, inputTweets = tweetIds, socialProofTypes = Some(utegSocialProofTypes) ) client.findTweetSocialProofs(socialProofRequest).map { result => val resultMap = result.socialProofResults.map(t => t.tweetId -> t).toMap tweetIds.map(resultMap.get) } } toRepositoryBatchWithView(lookup, chunkSize = 200) } @Provides @Singleton @Named(GraphTwoHopRepository) def providesGraphTwoHopRepository( clientId: ClientId, serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): KeyValueRepository[(Seq[Long], Long), Long, Seq[gfs.IntersectionValue]] = { val client = FinagleThriftClientBuilder .buildFinagleMethodPerEndpoint[gfs.Server.ServicePerEndpoint, gfs.Server.MethodPerEndpoint]( serviceIdentifier = serviceIdentifier, clientId = clientId, dest = "/s/cassowary/graph_feature_service-server", label = "gfs-repo", statsReceiver = statsReceiver, idempotency = Idempotent(1.percent), timeoutPerRequest = 350.milliseconds, timeoutTotal = 500.milliseconds ) def lookup( userIds: Seq[Long], viewerId: Long ): Future[Seq[Option[Seq[gfs.IntersectionValue]]]] = { val gfsIntersectionRequest = gfs.GfsPresetIntersectionRequest( userId = viewerId, candidateUserIds = userIds, presetFeatureTypes = gfs.PresetFeatureTypes.HtlTwoHop, intersectionIdLimit = Some(GFSInteractionIdsLimit) ) client .getPresetIntersection(gfsIntersectionRequest) .map { graphFeatureServiceResponse => val resultMap = graphFeatureServiceResponse.results .map(result => result.candidateUserId -> result.intersectionValues).toMap userIds.map(resultMap.get(_)) } } toRepositoryBatchWithView(lookup, chunkSize = 200) } @Provides @Singleton @Named(EarlybirdRepository) def providesEarlybirdSearchRepository( client: eb.EarlybirdService.MethodPerEndpoint, clientId: ClientId ): KeyValueRepository[EarlybirdQuery, Long, eb.ThriftSearchResult] = { def lookup( tweetIds: Seq[Long], viewerId: Long ): Future[Seq[Option[eb.ThriftSearchResult]]] = { val request = EarlybirdRequestUtil.getTweetsFeaturesRequest( userId = Some(viewerId), tweetIds = Some(tweetIds), clientId = Some(clientId.name), authorScoreMap = None, tensorflowModel = Some("timelines_unified_prod") ) client .search(request).map { response => val resultMap = response.searchResults .map(_.results.map { result => result.id -> result }.toMap).getOrElse(Map.empty) tweetIds.map(resultMap.get) } } toRepositoryBatchWithView(lookup) } protected def toRepository[K, V]( hydrate: K => Future[V] ): KeyValueRepository[Seq[K], K, V] = { def asRepository(keys: Seq[K]): Future[KeyValueResult[K, V]] = { Future.collect(keys.map(hydrate(_).liftToTry)).map { results => keys .zip(results) .foldLeft(new KeyValueResultBuilder[K, V]()) { case (bldr, (k, result)) => result match { case Return(v) => bldr.addFound(k, v) case _ => bldr.addNotFound(k) } }.result } } asRepository } protected def toRepositoryBatch[K, V]( hydrate: Seq[K] => Future[Seq[Option[V]]], chunkSize: Int = DefaultRPCChunkSize ): KeyValueRepository[Seq[K], K, V] = { def repository(keys: Seq[K]): Future[KeyValueResult[K, V]] = batchRepositoryProcess(keys, hydrate(keys)) KeyValueRepository.chunked(repository, ChunkingStrategy.equalSize(chunkSize)) } protected def toRepositoryBatchWithView[K, T, V]( hydrate: (Seq[K], T) => Future[Seq[Option[V]]], chunkSize: Int = DefaultRPCChunkSize ): KeyValueRepository[(Seq[K], T), K, V] = { def repository(input: (Seq[K], T)): Future[KeyValueResult[K, V]] = { val (keys, view) = input batchRepositoryProcess(keys, hydrate(keys, view)) } KeyValueRepository.chunked(repository, CustomChunkingStrategy.equalSizeWithView(chunkSize)) } private def batchRepositoryProcess[K, V]( keys: Seq[K], f: Future[Seq[Option[V]]] ): Future[KeyValueResult[K, V]] = { f.liftToTry .map { case Return(values) => keys .zip(values) .foldLeft(new KeyValueResultBuilder[K, V]()) { case (bldr, (k, value)) => value match { case Some(v) => bldr.addFound(k, v) case _ => bldr.addNotFound(k) } }.result case _ => keys .foldLeft(new KeyValueResultBuilder[K, V]()) { case (bldr, k) => bldr.addNotFound(k) }.result } } // Use only for cases not already covered by Servo's [[ChunkingStrategy]] object CustomChunkingStrategy { def equalSizeWithView[K, T](maxSize: Int): ((Seq[K], T)) => Seq[(Seq[K], T)] = { case (keys, view) => ChunkingStrategy .equalSize[K](maxSize)(keys) .map { chunk: Seq[K] => (chunk, view) } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TimelinesPersistenceStoreClientModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.inject.TwitterModule import com.twitter.inject.annotations.Flag import com.twitter.timelinemixer.clients.persistence.TimelinePersistenceManhattanClientBuilder import com.twitter.timelinemixer.clients.persistence.TimelinePersistenceManhattanClientConfig import com.twitter.timelinemixer.clients.persistence.TimelineResponseBatchesClient import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3 import com.twitter.util.Duration import javax.inject.Singleton object TimelinesPersistenceStoreClientModule extends TwitterModule { private val StagingDataset = "timeline_response_batches_v5_nonprod" private val ProdDataset = "timeline_response_batches_v5" private final val Timeout = "mh_persistence_store.timeout" flag[Duration](Timeout, 300.millis, "Timeout per request") @Provides @Singleton def providesTimelinesPersistenceStoreClient( @Flag(Timeout) timeout: Duration, injectedServiceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): TimelineResponseBatchesClient[TimelineResponseV3] = { val timelineResponseBatchesDataset = injectedServiceIdentifier.environment.toLowerCase match { case "prod" => ProdDataset case _ => StagingDataset } val timelineResponseBatchesConfig = new TimelinePersistenceManhattanClientConfig { val dataset = timelineResponseBatchesDataset val isReadOnly = false val serviceIdentifier = injectedServiceIdentifier override val defaultMaxTimeout = timeout override val maxRetryCount = 2 } TimelinePersistenceManhattanClientBuilder.buildTimelineResponseV3BatchesClient( timelineResponseBatchesConfig, statsReceiver ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TopicSocialProofClientModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout import com.twitter.inject.TwitterModule import com.twitter.strato.client.Client import com.twitter.timelines.clients.strato.topics.TopicSocialProofClient import com.twitter.timelines.clients.strato.topics.TopicSocialProofClientImpl import javax.inject.Named import javax.inject.Singleton object TopicSocialProofClientModule extends TwitterModule { @Singleton @Provides def providesSimilarityClient( @Named(BatchedStratoClientWithModerateTimeout) stratoClient: Client, statsReceiver: StatsReceiver ): TopicSocialProofClient = new TopicSocialProofClientImpl(stratoClient, statsReceiver) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TvWatchHistoryCacheClientModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.hashing.KeyHasher import com.twitter.home_mixer.param.HomeMixerInjectionNames.TvWatchedVideoIdsKeyCacheStore import com.twitter.inject.TwitterModule import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder import com.twitter.servo.cache.FinagleMemcache import com.twitter.servo.cache.KeyValueTransformingTtlCache import com.twitter.servo.cache.ObservableTtlCache import com.twitter.servo.cache.TtlCache import com.twitter.servo.util.Transformer import com.twitter.util.Try import java.nio.ByteBuffer import javax.inject.Named import javax.inject.Singleton import scala.collection.mutable.ArrayBuffer object TvWatchHistoryCacheClientModule extends TwitterModule { @Provides @Singleton @Named(TvWatchedVideoIdsKeyCacheStore) def providesEntityRealGraphClient( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver ): TtlCache[Long, Seq[Long]] = { val scopeName = "TvWatchedVideoIds" val memCacheClient = MemcachedClientBuilder.buildMemcachedClient( destName = "/srv#/prod/local/cache/related_videos:twemcaches", numTries = 1, numConnections = 1, requestTimeout = 50.milliseconds, globalTimeout = 100.milliseconds, connectTimeout = 100.milliseconds, acquisitionTimeout = 100.milliseconds, serviceIdentifier = serviceIdentifier, statsReceiver = statsReceiver, keyHasher = Some(KeyHasher.FNV1A_64) ) val longSeqTransformer: Transformer[Seq[Long], Array[Byte]] = new Transformer[Seq[Long], Array[Byte]] { def deserialize(input: Array[Byte]): ArrayBuffer[Long] = { val buffer = ByteBuffer.wrap(input) val resultBuffer = ArrayBuffer[Long]() while (buffer.hasRemaining) { val longValue = buffer.getLong() resultBuffer += longValue } resultBuffer } override def from(input: Array[Byte]): Try[Seq[Long]] = Try(deserialize(input)) override def to(b: Seq[Long]): Try[Array[Byte]] = ??? } val keyValueCache = new KeyValueTransformingTtlCache[Long, String, Seq[Long], Array[Byte]]( underlyingCache = new FinagleMemcache(memCacheClient), transformer = longSeqTransformer, underlyingKey = { key: Long => key.toString } ) ObservableTtlCache( underlyingCache = keyValueCache, statsReceiver = statsReceiver.scope(scopeName), windowSize = 1000, name = scopeName ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetWatchTimeMetadataModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetWatchTimeMetadataStore import com.twitter.home_mixer.store.TweetWatchTimeMetadataStore import com.twitter.inject.TwitterModule import com.twitter.storehaus.ReadableStore import com.twitter.twistly.thriftscala.VideoViewEngagementType import com.twitter.twistly.thriftscala.WatchTimeMetadata import javax.inject.Named import javax.inject.Singleton object TweetWatchTimeMetadataModule extends TwitterModule { @Provides @Singleton @Named(TweetWatchTimeMetadataStore) def providesTweetWatchTimeMetadataStore( tweetWatchTimeMetadataStore: TweetWatchTimeMetadataStore ): ReadableStore[(Long, VideoViewEngagementType), WatchTimeMetadata] = { tweetWatchTimeMetadataStore.tweetWatchTimeMetadataStore } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetyPieClientModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.thrift.ClientId import com.twitter.finagle.thriftmux.MethodBuilder import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient import com.twitter.inject.Injector import com.twitter.inject.annotations.Flags import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule import com.twitter.stitch.tweetypie.TweetyPie import com.twitter.tweetypie.thriftscala.TweetService import com.twitter.util.Duration import javax.inject.Singleton /** * Idempotent Tweetypie Thrift and Stitch client. */ object TweetypieClientModule extends ThriftMethodBuilderClientModule[ TweetService.ServicePerEndpoint, TweetService.MethodPerEndpoint ] with MtlsClient { private val TimeoutRequest = "tweetypie.timeout_request" private val TimeoutTotal = "tweetypie.timeout_total" flag[Duration](TimeoutRequest, 1000.millis, "Timeout per request") flag[Duration](TimeoutTotal, 1000.millis, "Total timeout") override val label: String = "tweetypie" override val dest: String = "/s/tweetypie/tweetypie" @Singleton @Provides def providesTweetypieStitchClient(tweetService: TweetService.MethodPerEndpoint): TweetyPie = new TweetyPie(tweetService) /** * TweetyPie client id must be in the form of {service.env} or it will not be treated as an * unauthorized client */ override protected def clientId(injector: Injector): ClientId = { val serviceIdentifier = injector.instance[ServiceIdentifier] ClientId(s"${serviceIdentifier.service}.${serviceIdentifier.environment}") } override protected def configureMethodBuilder( injector: Injector, methodBuilder: MethodBuilder ): MethodBuilder = { val timeoutRequest = injector.instance[Duration](Flags.named(TimeoutRequest)) val timeoutTotal = injector.instance[Duration](Flags.named(TimeoutTotal)) methodBuilder .withTimeoutPerRequest(timeoutRequest) .withTimeoutTotal(timeoutTotal) } override protected def sessionAcquisitionTimeout: Duration = 500.millis } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.google.inject.name.Named import com.twitter.conversions.DurationOps.RichDuration import com.twitter.cproxy.{thriftscala => cp} import com.twitter.deferredrpc.client.DeferredThriftService import com.twitter.deferredrpc.thrift.Datacenter import com.twitter.deferredrpc.thrift.DeferredRPC import com.twitter.deferredrpc.thrift.Target import com.twitter.finagle.ThriftMux import com.twitter.finagle.builder.ClientBuilder import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieStaticEntitiesCache import com.twitter.inject.TwitterModule import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder import com.twitter.servo.cache.ExpiringLruInProcessCache import com.twitter.servo.cache.FinagleMemcache import com.twitter.servo.cache.HotKeyMemcacheClient import com.twitter.servo.cache.KeyTransformer import com.twitter.servo.cache.KeyValueTransformingTtlCache import com.twitter.servo.cache.ObservableTtlCache import com.twitter.servo.cache.Serializer import com.twitter.servo.cache.ThriftSerializer import com.twitter.servo.cache.TtlCache import com.twitter.servo.util.Gate import com.twitter.timelinemixer.clients.rankedtweetcaching.CproxyTtlCacheWrapper import com.twitter.tweetypie.{thriftscala => tp} import javax.inject.Singleton import org.apache.thrift.protocol.TCompactProtocol object TweetypieStaticEntitiesCacheClientModule extends TwitterModule { private val ScopeName = "TweetypieStaticEntitiesMemcache" private val ProdDest = "/s/cache/timelinescorer_tweet_core_data:twemcaches" private val ProdKrpcDest = "/s/kafka-shared/krpc-server-cache" private val DevelKrpcDest = "/srv#/staging/local/kafka-shared/krpc-server-custdevel" private val tweetsSerializer: Serializer[tp.Tweet] = new ThriftSerializer[tp.Tweet](tp.Tweet, new TCompactProtocol.Factory()) private val keyTransformer: KeyTransformer[Long] = { tweetId => tweetId.toString } @Provides @Singleton @Named(TweetypieStaticEntitiesCache) def providesTweetypieStaticEntitiesCache( statsReceiver: StatsReceiver, serviceIdentifier: ServiceIdentifier ): TtlCache[Long, tp.Tweet] = { val memCacheClient = MemcachedClientBuilder.buildMemcachedClient( destName = ProdDest, numTries = 1, numConnections = 1, requestTimeout = 200.milliseconds, globalTimeout = 200.milliseconds, connectTimeout = 200.milliseconds, acquisitionTimeout = 200.milliseconds, serviceIdentifier = serviceIdentifier, statsReceiver = statsReceiver ) val hotkeyCacheClient = new HotKeyMemcacheClient( proxyClient = memCacheClient, inProcessCache = new ExpiringLruInProcessCache(ttl = 15.minute, maximumSize = 75000), statsReceiver = statsReceiver.scope(ScopeName).scope("inProcess") ) val krpcDest = serviceIdentifier.environment.toLowerCase match { case "prod" => ProdKrpcDest case _ => DevelKrpcDest } val deferredRpcClient = new DeferredRPC.FinagledClient( ClientBuilder() .name("deferredRpc") .dest(krpcDest) .requestTimeout(200.milliseconds) .hostConnectionLimit(3) .stack(ThriftMux.client.withMutualTls(serviceIdentifier)) .build() ) val cproxyClient = new cp.Cproxy.FinagledClient( service = new DeferredThriftService( deferredrpcService = deferredRpcClient, target = Target(Datacenter.AllOthers, "cproxy"), statsReceiver = statsReceiver.scope("deferredThriftService") ), stats = statsReceiver.scope("cproxy") ) val baseCache: KeyValueTransformingTtlCache[Long, String, tp.Tweet, Array[Byte]] = new KeyValueTransformingTtlCache( underlyingCache = new FinagleMemcache(hotkeyCacheClient), transformer = tweetsSerializer, underlyingKey = keyTransformer ) val observableTtlCache = ObservableTtlCache( underlyingCache = baseCache, statsReceiver = statsReceiver.scope(ScopeName), windowSize = 1000, name = ScopeName ) new CproxyTtlCacheWrapper[Long, tp.Tweet]( underlyingCache = observableTtlCache, cProxyCache = cproxyClient, cType = cp.CachePoolType.HomeTweetCoreData, valueSerializer = tweetsSerializer, keyTransformer = keyTransformer, replicationAvailable = Gate.True ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TwhinEmbeddingsModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinRebuildTweetEmbeddingsStore import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinRebuildUserPositiveEmbeddingsStore import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinTweetEmbeddingsStore import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserNegativeEmbeddingsStore import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserPositiveEmbeddingsStore import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinVideoEmbeddingsStore import com.twitter.home_mixer.store.TwhinEmbeddingsStore import com.twitter.inject.TwitterModule import com.twitter.simclusters_v2.{thriftscala => t} import com.twitter.storehaus.ReadableStore import javax.inject.Named import javax.inject.Singleton object TwhinEmbeddingsModule extends TwitterModule { @Provides @Singleton @Named(TwhinTweetEmbeddingsStore) def providesTwhinTweetEmbeddingFeaturesStore( twhinEmbeddingsStore: TwhinEmbeddingsStore ): ReadableStore[Long, t.TwhinTweetEmbedding] = twhinEmbeddingsStore.cachedTweetStore @Provides @Singleton @Named(TwhinVideoEmbeddingsStore) def providesTwhinVideoEmbeddingFeaturesStore( twhinEmbeddingsStore: TwhinEmbeddingsStore ): ReadableStore[Long, t.TwhinTweetEmbedding] = twhinEmbeddingsStore.cachedVideoStore @Provides @Singleton @Named(TwhinUserPositiveEmbeddingsStore) def providesTwhinUserPositiveEmbeddingFeaturesStore( twhinEmbeddingsStore: TwhinEmbeddingsStore ): ReadableStore[Long, t.TwhinTweetEmbedding] = twhinEmbeddingsStore.mhUserPositiveStore @Provides @Singleton @Named(TwhinRebuildUserPositiveEmbeddingsStore) def providesTwhinRebuildUserPositiveEmbeddingFeatureStore( twhinEmbeddingStore: TwhinEmbeddingsStore ): ReadableStore[(Long, Long), t.TwhinTweetEmbedding] = twhinEmbeddingStore.mhRebuildUserPositiveStore @Provides @Singleton @Named(TwhinUserNegativeEmbeddingsStore) def providesTwhinUserNegativeEmbeddingFeaturesStore( twhinEmbeddingsStore: TwhinEmbeddingsStore ): ReadableStore[Long, t.TwhinTweetEmbedding] = twhinEmbeddingsStore.mhUserNegativeStore @Provides @Singleton @Named(TwhinRebuildTweetEmbeddingsStore) def providesTwhinRebuildTweetEmbeddingFeaturesStore( twhinEmbeddingsStore: TwhinEmbeddingsStore ): ReadableStore[(Long, Long), t.TwhinTweetEmbedding] = twhinEmbeddingsStore.cachedTweetRebuildStore } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/UttTopicModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.escherbird.util.uttclient.CacheConfigV2 import com.twitter.escherbird.util.uttclient.CachedUttClientV2 import com.twitter.escherbird.util.uttclient.UttClientCacheConfigsV2 import com.twitter.escherbird.utt.strato.{thriftscala => utt} import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout import com.twitter.inject.TwitterModule import com.twitter.inject.annotations.Flag import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule.ConfigRepoLocalPath import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule.ServiceLocal import com.twitter.strato.client.Client import com.twitter.topiclisting.TopicListing import com.twitter.topiclisting.TopicListingBuilder import com.twitter.topiclisting.clients.utt.UttClient import com.twitter.topiclisting.utt.UttLocalization import com.twitter.topiclisting.utt.UttLocalizationImpl import com.twitter.tsp.stores.LocalizedUttRecommendableTopicsStore import com.twitter.tsp.stores.TopicStore import com.twitter.tsp.stores.UttTopicFilterStore import com.twitter.util.JavaTimer import javax.inject.Named import javax.inject.Singleton object UttTopicModule extends TwitterModule { private val StatsScope = "UttTopic" private val optOutStratoStorePath: String = "interests/optOutInterests" private val notInterestedInStorePath: String = "interests/notInterestedTopicsGetter" @Provides @Singleton def providesTopicListing( statsReceiver: StatsReceiver, @Flag(ServiceLocal) isServiceLocal: Boolean, @Flag(ConfigRepoLocalPath) localConfigRepoPath: String ): TopicListing = { val configSourceBasePath = if (isServiceLocal) Some(localConfigRepoPath) else None new TopicListingBuilder(statsReceiver, configSourceBasePath).build } @Provides @Singleton def providesUttLocalization( @Named(BatchedStratoClientWithModerateTimeout) stratoClient: Client, topicListing: TopicListing, statsReceiver: StatsReceiver ): UttLocalization = { // used the same capacity as in currently used client from TSP val defaultCacheConfigV2 = CacheConfigV2(capacity = 262143) val uttClientCacheConfigsV2 = UttClientCacheConfigsV2( getTaxonomyConfig = defaultCacheConfigV2, getUttTaxonomyConfig = defaultCacheConfigV2, getLeafIds = defaultCacheConfigV2, getLeafUttEntities = defaultCacheConfigV2 ) lazy val cachedUttClientV2 = new CachedUttClientV2( stratoClient = stratoClient, env = utt.Environment.Prod, cacheConfigs = uttClientCacheConfigsV2, statsReceiver = statsReceiver.scope("CachedUttClient") ) val uttClient = new UttClient(cachedUttClientV2, statsReceiver) new UttLocalizationImpl(topicListing, uttClient, statsReceiver.scope(StatsScope)) } @Provides @Singleton def providesUttTopicFilterStore( @Named(BatchedStratoClientWithModerateTimeout) stratoClient: Client, uttLocalization: UttLocalization, topicListing: TopicListing, statsReceiver: StatsReceiver ): UttTopicFilterStore = { val userOptOutTopicsStore = TopicStore.userOptOutTopicStore(stratoClient, optOutStratoStorePath)( statsReceiver.scope("interests_opt_out_store")) val explicitFollowingTopicsStore = TopicStore.explicitFollowingTopicStore(stratoClient)( statsReceiver.scope("explicit_following_interests_store")) val userNotInterestedInTopicsStore = TopicStore.notInterestedInTopicsStore(stratoClient, notInterestedInStorePath)( statsReceiver.scope("not_interested_in_store")) val localizedUttRecommendableTopicsStore = new LocalizedUttRecommendableTopicsStore( uttLocalization) val timer = new JavaTimer(isDaemon = true) new UttTopicFilterStore( topicListing = topicListing, userOptOutTopicsStore = userOptOutTopicsStore, explicitFollowingTopicsStore = explicitFollowingTopicsStore, notInterestedTopicsStore = userNotInterestedInTopicsStore, localizedUttRecommendableTopicsStore = localizedUttRecommendableTopicsStore, timer = timer, stats = statsReceiver.scope("UttTopicFilterStore") ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/module/VideoEmbeddingModule.scala ================================================ package com.twitter.home_mixer.module import com.google.inject.Provides import com.twitter.home_mixer.param.HomeMixerInjectionNames.VideoEmbeddingMHStore import com.twitter.home_mixer.store.VideoEmbeddingMHStore import com.twitter.inject.TwitterModule import com.twitter.media_understanding.video_summary.thriftscala.VideoEmbedding import com.twitter.storehaus.ReadableStore import javax.inject.Named import javax.inject.Singleton object VideoEmbeddingModule extends TwitterModule { @Provides @Singleton @Named(VideoEmbeddingMHStore) def providesVideoEmbeddingMHStore( videoEmbeddingMHStore: VideoEmbeddingMHStore ): ReadableStore[Long, VideoEmbedding] = { videoEmbeddingMHStore.videoEmbeddingMHStore } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/param/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/param/GlobalParamConfigModule.scala ================================================ package com.twitter.home_mixer.param import com.twitter.inject.TwitterModule import com.twitter.product_mixer.core.functional_component.configapi.registry.GlobalParamConfig object GlobalParamConfigModule extends TwitterModule { override def configure(): Unit = { bind[GlobalParamConfig].to[HomeGlobalParamConfig] } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.scala ================================================ package com.twitter.home_mixer.param import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableClipEmbeddingFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableClipEmbeddingMediaUnderstandingFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableDedupClusterId88FeatureHydrationParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableDedupClusterIdFeatureHydrationParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableGeoduckAuthorLocationHydatorParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableGrokVideoMetadataFeatureHydrationParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableImmersiveClientActionsClipEmbeddingQueryFeatureHydrationParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableImmersiveClientActionsQueryFeatureHydrationParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableLargeEmbeddingsFeatureHydrationParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableOnPremRealGraphQueryFeatures import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableRealGraphQueryFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableRealGraphViewerRelatedUsersFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableSimclustersSparseTweetFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTransformerPostEmbeddingJointBlueFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetLanguageFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetTextTokensEmbeddingFeatureScribingParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetVideoAggregatedWatchTimeFeatureScribingParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetypieContentFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetypieContentMediaEntityFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinRebuildTweetFeaturesOnlineParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinRebuildUserEngagementFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinRebuildUserPositiveFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinTweetFeaturesOnlineParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinUserNegativeFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinUserPositiveFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinVideoFeaturesOnlineParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinVideoFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableUserFavAvgTextEmbeddingsQueryFeatureParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableUserHistoryTransformerJointBlueEmbeddingFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableViewCountFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring import com.twitter.home_mixer.param.HomeGlobalParams._ import com.twitter.product_mixer.core.functional_component.configapi.registry.GlobalParamConfig import javax.inject.Inject import javax.inject.Singleton /** * Register Params that do not relate to a specific product. See GlobalParamConfig -> ParamConfig * for hooks to register Params based on type. */ @Singleton class HomeGlobalParamConfig @Inject() () extends GlobalParamConfig { override val booleanDeciderOverrides = Seq( EnableServedCandidateFeatureKeysKafkaPublishingParam, FeatureHydration.EnableSimClustersSimilarityFeaturesDeciderParam, FeatureHydration.EnableTweetypieContentFeaturesDeciderParam, FeatureHydration.EnableVideoSummaryEmbeddingFeatureDeciderParam, FeatureHydration.EnableVideoClipEmbeddingFeatureHydrationDeciderParam, FeatureHydration.EnableScoredVideoTweetsUserHistoryEventsQueryFeatureHydrationDeciderParam, FeatureHydration.EnableVideoClipEmbeddingMediaUnderstandingFeatureHydrationDeciderParam, EnableCommonFeaturesDataRecordCopyDuringPldrConversionParam, Scoring.UseSecondaryNaviClusterParam, Scoring.UseGPUNaviClusterTestUsersParam ) override val boundedDoubleDeciderOverrides = Seq(Scoring.NaviGPUBatchSizeParam) override val stringFSOverrides = Seq( Scoring.ModelNameParam, Scoring.ModelIdParam, Scoring.ProdModelIdParam, PostFeedbackPromptTitleParam, PostFeedbackPromptPositiveParam, PostFeedbackPromptNegativeParam, PostFeedbackPromptNeutralParam, ) override val booleanFSOverrides = Seq( AdsDisableInjectionBasedOnUserRoleParam, EnableTweetEntityServiceMigrationParam, EnableTweetEntityServiceVisibilityMigrationParam, EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, EnableSSPAdsBrandSafetySettingsFeatureHydratorParam, EnableDebugString, EnablePersistenceDebug, EnableLargeEmbeddingsFeatureHydrationParam, EnableNewTweetsPillAvatarsParam, EnableOnPremRealGraphQueryFeatures, EnableRealGraphQueryFeaturesParam, EnableRealGraphViewerRelatedUsersFeaturesParam, EnableScribeServedCandidatesParam, EnableSendScoresToClient, EnableSimclustersSparseTweetFeaturesParam, EnableCommunitiesContextParam, EnableSocialContextParam, EnableTwhinRebuildUserEngagementFeaturesParam, EnableTwhinRebuildUserPositiveFeaturesParam, EnableTwhinVideoFeaturesParam, EnableTwhinUserNegativeFeaturesParam, EnableTwhinUserPositiveFeaturesParam, EnableTwhinVideoFeaturesOnlineParam, EnableTwhinTweetFeaturesOnlineParam, EnableTwhinRebuildTweetFeaturesOnlineParam, EnableBasketballContextFeatureHydratorParam, EnablePostContextFeatureHydratorParam, EnableTransformerPostEmbeddingJointBlueFeaturesParam, EnableClipEmbeddingFeaturesParam, EnableClipEmbeddingMediaUnderstandingFeaturesParam, EnableUserHistoryTransformerJointBlueEmbeddingFeaturesParam, EnableLandingPage, EnableTenSecondsLogicForVQV, EnableImmersiveVQV, EnableExploreSimclustersLandingPage, EnableTweetLanguageFeaturesParam, EnableTweetypieContentFeaturesParam, EnableTweetypieContentMediaEntityFeaturesParam, EnableViewCountFeaturesParam, ListMandarinTweetsParams.ListMandarinTweetsEnable, Scoring.EnableNoNegHeuristicParam, Scoring.EnableNegSectionRankingParam, Scoring.EnableBinarySchemeForVQVParam, Scoring.EnableBinarySchemeForDwellParam, Scoring.EnableDwellOrVQVParam, Scoring.UseRealtimeNaviClusterParam, Scoring.UseGPUNaviClusterParam, Scoring.RequestNormalizedScoresParam, Scoring.AddNoiseInWeightsPerLabel, Scoring.EnableDailyFrozenNoisyWeights, Scoring.TwhinDiversityRescoringParam, Scoring.CategoryDiversityRescoringParam, Scoring.UseVideoNaviClusterParam, Scoring.NormalizedNegativeHead, Scoring.UseWeightForNegHeadParam, Scoring.ConstantNegativeHead, Scoring.UseProdInPhoenixParams.EnableProdDwellForPhoenixParam, Scoring.UseProdInPhoenixParams.EnableProdFavForPhoenixParam, Scoring.UseProdInPhoenixParams.EnableProdGoodClickV1ForPhoenixParam, Scoring.UseProdInPhoenixParams.EnableProdGoodClickV2ForPhoenixParam, Scoring.UseProdInPhoenixParams.EnableProdNegForPhoenixParam, Scoring.UseProdInPhoenixParams.EnableProdProfileClickForPhoenixParam, Scoring.UseProdInPhoenixParams.EnableProdReplyForPhoenixParam, Scoring.UseProdInPhoenixParams.EnableProdRetweetForPhoenixParam, Scoring.UseProdInPhoenixParams.EnableProdShareForPhoenixParam, Scoring.UseProdInPhoenixParams.EnableProdVQVForPhoenixParam, Scoring.UseProdInPhoenixParams.EnableProdOpenLinkForPhoenixParam, Scoring.UseProdInPhoenixParams.EnableProdScreenshotForPhoenixParam, Scoring.UseProdInPhoenixParams.EnableProdBookmarkForPhoenixParam, EnableUserFavAvgTextEmbeddingsQueryFeatureParam, EnableTweetTextTokensEmbeddingFeatureScribingParam, EnableTweetVideoAggregatedWatchTimeFeatureScribingParam, EnableImmersiveClientActionsQueryFeatureHydrationParam, EnableImmersiveClientActionsClipEmbeddingQueryFeatureHydrationParam, EnableGrokVideoMetadataFeatureHydrationParam, EnableDedupClusterIdFeatureHydrationParam, EnableDedupClusterId88FeatureHydrationParam, EnableServedFilterAllRequests, EnablePinnedTweetsCarouselParam, EnablePostDetailsNegativeFeedbackParam, EnablePostFeedbackParam, EnablePostFollowupParam, EnableSlopFilter, EnableNsfwFilter, EnableSoftNsfwFilter, EnableGrokGoreFilter, EnableMinVideoDurationFilter, EnableMaxVideoDurationFilter, EnableGrokSpamFilter, EnableGrokViolentFilter, EnableClusterBasedDedupFilter, EnableCountryFilter, EnableRegionFilter, EnableHasMultipleMediaFilter, EnableClusterBased88DedupFilter, EnableNoClusterFilter, EnableSlopFilterLowSignalUsers, EnableSlopFilterEligibleUserStateParam, EnableGrokAnnotations, EnableTopicBasedRealTimeAggregateFeatureHydratorParam, EnableTopicCountryBasedRealTimeAggregateFeatureHydratorParam, EnableTopicEdgeAggregateFeatureHydratorParam, EnableAdditionalChildFeedbackParam, EnableGeoduckAuthorLocationHydatorParam, EnableTweetRTAMhOnlyParam, EnableTweetRTAMhFallbackParam, EnableTweetCountryRTAMhOnlyParam, EnableTweetCountryRTAMhFallbackParam, EnableUserRTAMhOnlyParam, EnableUserRTAMhFallbackParam, EnableUserAuthorRTAMhOnlyParam, EnableUserAuthorRTAMhFallbackParam, EnableBlockMuteReportChildFeedbackParam, EnablePhoenixScorerParam, EnableUserActionsShadowScribeParam, ) override val boundedIntFSOverrides = Seq( MaxNumberReplaceInstructionsParam, TimelinesPersistenceStoreMaxEntriesPerClient, ExcludeServedTweetIdsNumberParam, IsSelectedByHeavyRankerCountParam, SlopMinFollowers, UserActionsMaxCount, PhoenixTimeoutInMsParam, ) override val boundedLongFSOverrides = Seq( DedupHistoricalEventsTimeWindowParam, MinVideoDurationThresholdParam, MaxVideoDurationThresholdParam) override val boundedDoubleFSOverrides = Seq( SlopMaxScore, PostFeedbackThresholdParam, PostFollowupThresholdParam, // Model Weights Scoring.ModelWeights.BookmarkParam, Scoring.ModelWeights.Dwell0Param, Scoring.ModelWeights.Dwell1Param, Scoring.ModelWeights.Dwell2Param, Scoring.ModelWeights.Dwell3Param, Scoring.ModelWeights.Dwell4Param, Scoring.ModelWeights.DwellParam, Scoring.ModelWeights.FavParam, Scoring.ModelWeights.GoodClickParam, Scoring.ModelWeights.GoodClickV1Param, Scoring.ModelWeights.GoodClickV2Param, Scoring.ModelWeights.GoodProfileClickParam, Scoring.ModelWeights.NegativeFeedbackV2Param, Scoring.ModelWeights.OpenLinkParam, Scoring.ModelWeights.ProfileDwelledParam, Scoring.ModelWeights.ReplyEngagedByAuthorParam, Scoring.ModelWeights.ReplyParam, Scoring.ModelWeights.ReportParam, Scoring.ModelWeights.RetweetParam, Scoring.ModelWeights.ScreenshotParam, Scoring.ModelWeights.ShareMenuClickParam, Scoring.ModelWeights.ShareParam, Scoring.ModelWeights.StrongNegativeFeedbackParam, Scoring.ModelWeights.TweetDetailDwellParam, Scoring.ModelWeights.VideoPlayback50Param, Scoring.ModelWeights.VideoQualityViewImmersiveParam, Scoring.ModelWeights.VideoQualityViewParam, Scoring.ModelWeights.VideoQualityWatchParam, Scoring.ModelWeights.VideoWatchTimeMsParam, Scoring.ModelWeights.WeakNegativeFeedbackParam, // Model Biases Scoring.ModelBiases.VideoQualityViewParam, Scoring.ModelBiases.VideoQualityViewImmersiveParam, Scoring.ModelBiases.VideoQualityWatchParam, Scoring.NoisyWeightAlphaParam, Scoring.NoisyWeightBetaParam, Scoring.NegativeScoreConstantFilterThresholdParam, Scoring.NegativeScoreNormFilterThresholdParam, Scoring.RequestRankDecayFactorParam, Scoring.ScoreThresholdForVQVParam, Scoring.ScoreThresholdForDwellParam, Scoring.BinarySchemeConstantForVQVParam, Scoring.ImpressedMediaClusterBasedRescoringParam, // ModelDebiases Scoring.ModelDebiases.FavParam, Scoring.ModelDebiases.ReplyParam, Scoring.ModelDebiases.RetweetParam, Scoring.ModelDebiases.GoodClickV1Param, Scoring.ModelDebiases.GoodClickV2Param, Scoring.ModelDebiases.GoodProfileClickParam, Scoring.ModelDebiases.ReplyEngagedByAuthorParam, Scoring.ModelDebiases.VideoQualityViewParam, Scoring.ModelDebiases.VideoQualityViewImmersiveParam, Scoring.ModelDebiases.NegativeFeedbackV2Param, Scoring.ModelDebiases.BookmarkParam, Scoring.ModelDebiases.ShareParam, Scoring.ModelDebiases.DwellParam, Scoring.ModelDebiases.VideoQualityWatchParam, Scoring.ModelDebiases.VideoWatchTimeMsParam ) override val longSetFSOverrides = Seq( RateLimitTestIdsParam, BasketballTeamAccountIdsParam, Scoring.AuthorListForDataCollectionParam ) override val boundedDurationFSOverrides = Seq( FeedbackFatigueFilteringDurationParam, ExcludeServedTweetIdsDurationParam, ExcludeServedAuthorIdsDurationParam ) override val enumFSOverrides = Seq( PhoenixInferenceClusterParam ) override val longSeqFSOverrides = Seq( ListMandarinTweetsParams.ListMandarinTweetsLists ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.scala ================================================ package com.twitter.home_mixer.param import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.param.decider.DeciderKey import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.timelines.configapi.decider.BooleanDeciderParam import com.twitter.timelines.configapi.decider.DeciderBoundedParam import com.twitter.util.Duration /** * Instantiate Params that do not relate to a specific product. * * @see [[com.twitter.product_mixer.core.product.ProductParamConfig.supportedClientFSName]] */ object HomeGlobalParams { /** * This param is used to disable ads injection for timelines served by home-mixer. * It is currently used to maintain user-role based no-ads lists for automation accounts, * and should NOT be used for other purposes. */ object AdsDisableInjectionBasedOnUserRoleParam extends FSParam( name = "home_mixer_ads_disable_injection_based_on_user_role", default = false ) object EnableTweetEntityServiceMigrationParam extends FSParam[Boolean]( name = "home_mixer_enable_tweet_entity_service_migration", default = false ) object EnableTweetEntityServiceVisibilityMigrationParam extends FSParam[Boolean]( name = "home_mixer_enable_tweet_entity_service_visibility_migration", default = false ) object EnableSendScoresToClient extends FSParam[Boolean]( name = "home_mixer_enable_send_scores_to_client", default = false ) object EnableDebugString extends FSParam[Boolean]( name = "home_mixer_enable_debug_string", default = false ) object EnablePersistenceDebug extends FSParam[Boolean]( name = "home_mixer_enable_persistence_debug", default = false ) object MaxNumberReplaceInstructionsParam extends FSBoundedParam[Int]( name = "home_mixer_max_number_replace_instructions", default = 10, min = 1, max = 20 ) object TimelinesPersistenceStoreMaxEntriesPerClient extends FSBoundedParam[Int]( name = "home_mixer_timelines_persistence_store_max_entries_per_client", default = 1800, min = 500, max = 5000 ) object EnableNewTweetsPillAvatarsParam extends FSParam[Boolean]( name = "home_mixer_enable_new_tweets_pill_avatars", default = true ) object EnableSocialContextParam extends FSParam[Boolean]( name = "home_mixer_enable_social_context", default = false ) object EnableCommunitiesContextParam extends FSParam[Boolean]( name = "home_mixer_enable_communities_context", default = true ) object EnableAdvertiserBrandSafetySettingsFeatureHydratorParam extends FSParam[Boolean]( name = "home_mixer_enable_advertiser_brand_safety_settings_feature_hydrator", default = true ) object EnableBasketballContextFeatureHydratorParam extends FSParam[Boolean]( name = "home_mixer_enable_basketball_context_feature_hydrator", default = false ) object EnablePostContextFeatureHydratorParam extends FSParam[Boolean]( name = "home_mixer_enable_post_context_feature_hydrator", default = false ) object BasketballTeamAccountIdsParam extends FSParam[Set[Long]]( name = "home_mixer_basketball_team_account_ids", default = Set() ) object EnableSSPAdsBrandSafetySettingsFeatureHydratorParam extends FSParam[Boolean]( name = "home_mixer_enable_ssp_ads_brand_safety_settings_feature_hydrator", default = true ) object ExcludeServedTweetIdsNumberParam extends FSBoundedParam[Int]( name = "home_mixer_exclude_served_tweet_ids_number", default = 100, min = 0, max = 100 ) object ExcludeServedTweetIdsDurationParam extends FSBoundedParam[Duration]( "home_mixer_exclude_served_tweet_ids_in_minutes", default = 10.minutes, min = 1.minute, max = 60.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object ExcludeServedAuthorIdsDurationParam extends FSBoundedParam[Duration]( "home_mixer_exclude_served_author_ids_in_minutes", default = 60.minutes, min = 1.minute, max = 60.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object EnableServedFilterAllRequests extends FSParam[Boolean]( name = "home_mixer_enable_served_filter_all_requests", default = false ) object EnableScribeServedCandidatesParam extends FSParam[Boolean]( name = "home_mixer_served_tweets_enable_scribing", default = false ) object EnableServedCandidateFeatureKeysKafkaPublishingParam extends BooleanDeciderParam( decider = DeciderKey.EnableServedCandidateFeatureKeysKafkaPublishing) object RateLimitTestIdsParam extends FSParam[Set[Long]]( name = "home_mixer_rate_limit_test_ids", default = Set.empty ) object IsSelectedByHeavyRankerCountParam extends FSBoundedParam[Int]( name = "home_mixer_is_selected_by_heavy_ranker_count", default = 100, min = 0, max = 2000 ) object EnableAdditionalChildFeedbackParam extends FSParam[Boolean]( name = "home_mixer_enable_additional_child_feedback", default = false ) object EnableBlockMuteReportChildFeedbackParam extends FSParam[Boolean]( name = "home_mixer_enable_block_mute_report_child_feedback", default = false ) object ListMandarinTweetsParams { object ListMandarinTweetsEnable extends FSParam[Boolean]( name = "home_mixer_mandarin_list_tweets_enabled", default = false ) object ListMandarinTweetsLists extends FSParam[Seq[Long]]( name = "home_mixer_mandarin_tweets_lists", default = Seq.empty ) } object FeatureHydration { object EnableLargeEmbeddingsFeatureHydrationParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_large_embeddings", default = false ) object EnableSimClustersSimilarityFeaturesDeciderParam extends BooleanDeciderParam( decider = DeciderKey.EnableSimClustersSimilarityFeatureHydration ) object EnableOnPremRealGraphQueryFeatures extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_on_prem_real_graph_query_features", default = false ) object EnableRealGraphQueryFeaturesParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_real_graph_query_features", default = false ) object EnableRealGraphViewerRelatedUsersFeaturesParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_real_graph_viewer_related_users_features", default = false ) object EnableSimclustersSparseTweetFeaturesParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_simclusters_sparse_tweet_features", default = false ) object EnableTwhinUserPositiveFeaturesParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_twhin_user_positive_features", default = false ) object EnableTwhinVideoFeaturesParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_twhin_video_features", default = false ) object EnableTwhinUserNegativeFeaturesParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_twhin_user_negative_features", default = false ) object EnableTwhinVideoFeaturesOnlineParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_twhin_video_online_features", default = false ) object EnableTwhinRebuildUserEngagementFeaturesParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_twhin_rebuild_user_engagement_features", default = false ) object EnableTwhinRebuildUserPositiveFeaturesParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_twhin_rebuild_user_positive_features", default = false ) object EnableClipEmbeddingFeaturesParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_clip_embedding_features", default = false ) object EnableClipEmbeddingMediaUnderstandingFeaturesParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_clip_embedding_media_understanding_features", default = false ) object EnableUserHistoryTransformerJointBlueEmbeddingFeaturesParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_user_history_transformer_joint_blue_embedding_features", default = false ) object EnableTweetLanguageFeaturesParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_tweet_language_features", default = false ) object EnableTwhinTweetFeaturesOnlineParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_twhin_tweet_online_features", default = false ) object EnableTwhinRebuildTweetFeaturesOnlineParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_twhin_rebuild_tweet_online_features", default = false ) object EnableTransformerPostEmbeddingJointBlueFeaturesParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_transformer_post_embedding_features_joint_blue", default = false ) object EnableTweetypieContentFeaturesDeciderParam extends BooleanDeciderParam( decider = DeciderKey.EnableTweetypieContentFeatures ) object EnableTweetypieContentFeaturesParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_tweetypie_content_features", default = true ) object EnableTweetypieContentMediaEntityFeaturesParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_tweetypie_content_media_entity_features", default = true ) object EnableUserFavAvgTextEmbeddingsQueryFeatureParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_user_fav_avg_text_embeddings_query_feature", default = false ) object EnableTweetTextTokensEmbeddingFeatureScribingParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_tweet_text_tokens_embedding_feature_scribing", default = false ) object EnableTweetVideoAggregatedWatchTimeFeatureScribingParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_tweet_video_aggregated_watch_time", default = false ) object EnableImmersiveClientActionsQueryFeatureHydrationParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_immersive_client_actions", default = false ) object EnableImmersiveClientActionsClipEmbeddingQueryFeatureHydrationParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_immersive_client_actions_clip_embedding", default = false ) object EnableGrokVideoMetadataFeatureHydrationParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_grok_video_metadata", default = false ) object EnableDedupClusterIdFeatureHydrationParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_dedup_cluster_id", default = false ) object EnableDedupClusterId88FeatureHydrationParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_dedup_cluster_id_88", default = false ) object EnableGeoduckAuthorLocationHydatorParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_geoduck_author_location_hydrator", default = false ) object EnableViewCountFeaturesParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_view_count_features", default = false ) object EnableVideoSummaryEmbeddingFeatureDeciderParam extends BooleanDeciderParam( decider = DeciderKey.EnableVideoSummaryEmbeddingHydration ) object EnableVideoClipEmbeddingFeatureHydrationDeciderParam extends BooleanDeciderParam( decider = DeciderKey.EnableVideoClipEmbeddingHydration ) object EnableScoredVideoTweetsUserHistoryEventsQueryFeatureHydrationDeciderParam extends BooleanDeciderParam( decider = DeciderKey.EnableScoredVideoTweetsUserHistoryEventsQueryFeatureHydrationDeciderParam ) object EnableVideoClipEmbeddingMediaUnderstandingFeatureHydrationDeciderParam extends BooleanDeciderParam( decider = DeciderKey.EnableVideoClipEmbeddingMediaUnderstandingHydration ) } object Scoring { object AuthorListForDataCollectionParam extends FSParam[Set[Long]]( name = "home_mixer_author_list_for_data_collection", default = Set.empty[Long] ) object ModelNameParam extends FSParam[String]( name = "home_mixer_model_name", default = "" ) object ImpressedMediaClusterBasedRescoringParam extends FSBoundedParam[Double]( name = "home_mixer_impressed_media_cluster_based_rescoring", default = 0.0, min = 0.0, max = 0.2 ) object ImpressedImageClusterBasedRescoringParam extends FSBoundedParam[Double]( name = "home_mixer_impressed_image_cluster_based_rescoring", default = 0.0, min = 0.0, max = 1.0 ) object ModelIdParam extends FSParam[String]( name = "home_mixer_model_id", default = "Home" ) object ProdModelIdParam extends FSParam[String]( name = "home_mixer_model_prod_model_id", default = "Home" ) object UseRealtimeNaviClusterParam extends FSParam[Boolean]( name = "home_mixer_model_use_realtime_navi_cluster", default = false ) object UseGPUNaviClusterParam extends FSParam[Boolean]( name = "home_mixer_model_use_gpu_navi_cluster", default = false ) object UseSecondaryNaviClusterParam extends BooleanDeciderParam(decider = DeciderKey.EnableSecondaryNaviRecapCluster) object UseGPUNaviClusterTestUsersParam extends BooleanDeciderParam(decider = DeciderKey.EnableGPUNaviRecapClusterTestUsers) object UseVideoNaviClusterParam extends FSParam[Boolean]("home_mixer_model_use_video_navi_cluster", false) object NaviGPUBatchSizeParam extends DeciderBoundedParam[Double]( decider = DeciderKey.NaviGPUClusterRequestBatchSize, default = 1800.0, min = 0.0, max = 10000.0 ) object AddNoiseInWeightsPerLabel extends FSParam[Boolean]( name = "home_mixer_add_noise_in_weights_per_label", default = false ) object EnableDailyFrozenNoisyWeights extends FSParam[Boolean]( name = "home_mixer_enable_daily_frozen_weights", default = false ) object NoisyWeightAlphaParam extends FSBoundedParam[Double]( name = "home_mixer_noisy_weight_alpha_param", default = 2, min = 0.0, max = 10.0 ) object NoisyWeightBetaParam extends FSBoundedParam[Double]( name = "home_mixer_noisy_weight_beta_param", default = 2, min = 0.0, max = 10.0 ) object NegativeScoreConstantFilterThresholdParam extends FSBoundedParam[Double]( name = "home_mixer_negative_score_constant_filter_threshold", default = 1e-3, min = 0, max = 1 ) object NegativeScoreNormFilterThresholdParam extends FSBoundedParam[Double]( name = "home_mixer_negative_score_norm_filter_threshold", default = 0.15, min = 0, max = 1 ) object RequestNormalizedScoresParam extends FSParam[Boolean]( name = "home_mixer_request_normalized_scores", default = false ) object NormalizedNegativeHead extends FSParam[Boolean]( name = "home_mixer_normalized_negative_head", default = false ) object UseWeightForNegHeadParam extends FSParam[Boolean]( name = "home_mixer_use_weight_for_neg_head", default = false ) object ConstantNegativeHead extends FSParam[Boolean]( name = "home_mixer_constant_negative_head", default = false ) object EnableNoNegHeuristicParam extends FSParam[Boolean]( name = "home_mixer_no_neg_heuristic", default = false ) object EnableNegSectionRankingParam extends FSParam[Boolean]( name = "home_mixer_neg_section_ranking", default = false ) object RequestRankDecayFactorParam extends FSBoundedParam[Double]( name = "home_mixer_request_rank_decay_factor", default = 0.95, min = 0, max = 1 ) object ScoreThresholdForVQVParam extends FSBoundedParam[Double]( name = "home_mixer_score_threshold_for_vqv", default = 0.0, min = 0.0, max = 1.0 ) object ScoreThresholdForDwellParam extends FSBoundedParam[Double]( name = "home_mixer_score_threshold_for_dwell", default = 0.0, min = 0.0, max = 1.0 ) object EnableBinarySchemeForVQVParam extends FSParam[Boolean]( name = "home_mixer_enable_binary_scheme_for_vqv", default = false ) object BinarySchemeConstantForVQVParam extends FSBoundedParam[Double]( name = "home_mixer_constant_binary_scheme_for_vqv", default = 0.0, min = 0.0, max = 1.0 ) object EnableBinarySchemeForDwellParam extends FSParam[Boolean]( name = "home_mixer_enable_binary_scheme_for_dwell", default = false ) object EnableDwellOrVQVParam extends FSParam[Boolean]( name = "home_mixer_enable_dwell_or_video_watch_time", default = false ) object TwhinDiversityRescoringParam extends FSParam[Boolean]( name = "home_mixer_twhin_diversity_rescoring", default = false ) object CategoryDiversityRescoringParam extends FSParam[Boolean]( name = "home_mixer_category_diversity_rescoring", default = false ) object ModelBiases { object VideoQualityViewParam extends FSBoundedParam[Double]( name = "home_mixer_model_bias_video_quality_viewed", default = 0.0, min = 0.0, max = 100.0 ) object VideoQualityViewImmersiveParam extends FSBoundedParam[Double]( name = "home_mixer_model_bias_video_quality_viewed_immersive", default = 0.0, min = 0.0, max = 100.0 ) object VideoQualityWatchParam extends FSBoundedParam[Double]( name = "home_mixer_model_bias_video_quality_watched", default = 0.0, min = 0.0, max = 100.0 ) } object ModelDebiases { object FavParam extends FSBoundedParam[Double]( name = "home_mixer_model_debias_fav", default = 0.0, min = -10000.0, max = 10000.0 ) object RetweetParam extends FSBoundedParam[Double]( name = "home_mixer_model_debias_retweet", default = 0.0, min = -10000.0, max = 10000.0 ) object ReplyParam extends FSBoundedParam[Double]( name = "home_mixer_model_debias_reply", default = 0.0, min = -10000.0, max = 10000.0 ) object DwellParam extends FSBoundedParam[Double]( name = "home_mixer_model_debias_dwell", default = 0.0, min = -10000.0, max = 10000.0 ) object GoodProfileClickParam extends FSBoundedParam[Double]( name = "home_mixer_model_debias_good_profile_click", default = 0.0, min = -10000.0, max = 10000.0 ) object VideoWatchTimeMsParam extends FSBoundedParam[Double]( name = "home_mixer_model_debias_video_watch_time_ms", default = 0.0, min = -10000.0, max = 10000.0 ) object VideoQualityViewParam extends FSBoundedParam[Double]( name = "home_mixer_model_debias_video_quality_viewed", default = 0.0, min = -10000.0, max = 10000.0 ) object VideoQualityViewImmersiveParam extends FSBoundedParam[Double]( name = "home_mixer_model_debias_video_quality_viewed_immersive", default = 0.0, min = -10000.0, max = 10000.0 ) object ReplyEngagedByAuthorParam extends FSBoundedParam[Double]( name = "home_mixer_model_debias_reply_engaged_by_author", default = 0.0, min = -10000.0, max = 10000.0 ) object GoodClickV1Param extends FSBoundedParam[Double]( name = "home_mixer_model_debias_good_click_v1", default = 0.0, min = -10000.0, max = 10000.0 ) object GoodClickV2Param extends FSBoundedParam[Double]( name = "home_mixer_model_debias_good_click_v2", default = 0.0, min = -10000.0, max = 10000.0 ) object BookmarkParam extends FSBoundedParam[Double]( name = "home_mixer_model_debias_bookmark", default = 0.0, min = -10000.0, max = 10000.0 ) object ShareParam extends FSBoundedParam[Double]( name = "home_mixer_model_debias_share", default = 0.0, min = -10000.0, max = 10000.0 ) object NegativeFeedbackV2Param extends FSBoundedParam[Double]( name = "home_mixer_model_debias_negative_feedback_v2", default = 0.0, min = -10000.0, max = 10000.0 ) object VideoQualityWatchParam extends FSBoundedParam[Double]( name = "home_mixer_model_debias_video_quality_watched", default = 0.0, min = -10000.0, max = 10000.0 ) } object ModelWeights { object FavParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_fav", default = 0.0, min = -10000.0, max = 10000.0 ) object RetweetParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_retweet", default = 0.0, min = -10000.0, max = 10000.0 ) object ReplyParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_reply", default = 0.0, min = -10000.0, max = 10000.0 ) object GoodProfileClickParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_good_profile_click", default = 0.0, min = -10000.0, max = 10000.0 ) object VideoPlayback50Param extends FSBoundedParam[Double]( name = "home_mixer_model_weight_video_playback50", default = 0.0, min = -10000.0, max = 10000.0 ) object VideoQualityViewParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_video_quality_viewed", default = 0.0, min = -10000.0, max = 10000.0 ) object VideoQualityViewImmersiveParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_video_quality_viewed_immersive", default = 0.0, min = -10000.0, max = 10000.0 ) object ReplyEngagedByAuthorParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_reply_engaged_by_author", default = 0.0, min = -10000.0, max = 10000.0 ) object GoodClickParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_good_click", default = 0.0, min = -10000.0, max = 10000.0 ) object GoodClickV1Param extends FSBoundedParam[Double]( name = "home_mixer_model_weight_good_click_v1", default = 0.0, min = -10000.0, max = 10000.0 ) object GoodClickV2Param extends FSBoundedParam[Double]( name = "home_mixer_model_weight_good_click_v2", default = 0.0, min = -10000.0, max = 10000.0 ) object TweetDetailDwellParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_tweet_detail_dwell", default = 0.0, min = -10000.0, max = 10000.0 ) object ProfileDwelledParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_profile_dwelled", default = 0.0, min = -10000.0, max = 10000.0 ) object BookmarkParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_bookmark", default = 0.0, min = -10000.0, max = 10000.0 ) object ShareParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_share", default = 0.0, min = -10000.0, max = 10000.0 ) object ShareMenuClickParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_share_menu_click", default = 0.0, min = -10000.0, max = 10000.0 ) object NegativeFeedbackV2Param extends FSBoundedParam[Double]( name = "home_mixer_model_weight_negative_feedback_v2", default = 0.0, min = -10000.0, max = 10000.0 ) object ReportParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_report", default = 0.0, min = -20000.0, max = 0.0 ) object WeakNegativeFeedbackParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_weak_negative_feedback", default = 0.0, min = -1000.0, max = 0.0 ) object StrongNegativeFeedbackParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_strong_negative_feedback", default = 0.0, min = -1000.0, max = 0.0 ) object DwellParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_dwell", default = 0.0, min = -10000.0, max = 10000.0 ) object OpenLinkParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_open_link", default = 0.0, min = -10000.0, max = 10000.0 ) object ScreenshotParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_screenshot", default = 0.0, min = -10000.0, max = 10000.0 ) object VideoWatchTimeMsParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_video_watch_time_ms", default = 0.0, min = -10000.0, max = 10000.0 ) // Categorical Dwell Params object Dwell0Param extends FSBoundedParam[Double]( name = "home_mixer_model_weight_dwell_0", default = 0.0, min = 0.0, max = 1000.0 ) object Dwell1Param extends FSBoundedParam[Double]( name = "home_mixer_model_weight_dwell_1", default = 0.0, min = 0.0, max = 1000.0 ) object Dwell2Param extends FSBoundedParam[Double]( name = "home_mixer_model_weight_dwell_2", default = 0.0, min = 0.0, max = 1000.0 ) object Dwell3Param extends FSBoundedParam[Double]( name = "home_mixer_model_weight_dwell_3", default = 0.0, min = 0.0, max = 1000.0 ) object Dwell4Param extends FSBoundedParam[Double]( name = "home_mixer_model_weight_dwell_4", default = 0.0, min = 0.0, max = 1000.0 ) object VideoQualityWatchParam extends FSBoundedParam[Double]( name = "home_mixer_model_weight_video_quality_watched", default = 0.0, min = -10000.0, max = 10000.0 ) } object UseProdInPhoenixParams { object EnableProdFavForPhoenixParam extends FSParam[Boolean]( name = "home_mixer_enable_prod_fav_for_phoenix", default = false ) object EnableProdReplyForPhoenixParam extends FSParam[Boolean]( name = "home_mixer_enable_prod_reply_for_phoenix", default = false ) object EnableProdShareForPhoenixParam extends FSParam[Boolean]( name = "home_mixer_enable_prod_share_for_phoenix", default = false ) object EnableProdRetweetForPhoenixParam extends FSParam[Boolean]( name = "home_mixer_enable_prod_retweet_for_phoenix", default = false ) object EnableProdVQVForPhoenixParam extends FSParam[Boolean]( name = "home_mixer_enable_prod_vqv_for_phoenix", default = false ) object EnableProdDwellForPhoenixParam extends FSParam[Boolean]( name = "home_mixer_enable_prod_dwell_for_phoenix", default = false ) object EnableProdNegForPhoenixParam extends FSParam[Boolean]( name = "home_mixer_enable_prod_neg_for_phoenix", default = false ) object EnableProdProfileClickForPhoenixParam extends FSParam[Boolean]( name = "home_mixer_enable_prod_profile_click_for_phoenix", default = false ) object EnableProdGoodClickV1ForPhoenixParam extends FSParam[Boolean]( name = "home_mixer_enable_prod_good_click_v1_for_phoenix", default = false ) object EnableProdGoodClickV2ForPhoenixParam extends FSParam[Boolean]( name = "home_mixer_enable_prod_good_click_v2_for_phoenix", default = false ) object EnableProdOpenLinkForPhoenixParam extends FSParam[Boolean]( name = "home_mixer_enable_prod_open_link_for_phoenix", default = false ) object EnableProdScreenshotForPhoenixParam extends FSParam[Boolean]( name = "home_mixer_enable_prod_screenshot_for_phoenix", default = false ) object EnableProdBookmarkForPhoenixParam extends FSParam[Boolean]( name = "home_mixer_enable_prod_bookmark_for_phoenix", default = false ) } } object EnableTenSecondsLogicForVQV extends FSParam[Boolean]( name = "home_mixer_enable_ten_seconds_logic_for_vqv", default = true ) object EnableImmersiveVQV extends FSParam[Boolean]( name = "home_mixer_enable_immersive_vqv", default = false ) object EnableLandingPage extends FSParam[Boolean]( name = "home_mixer_enable_landing_page", default = false ) object EnableExploreSimclustersLandingPage extends FSParam[Boolean]( name = "home_mixer_enable_explore_simclusters_landing_page", default = false ) object EnableTopicBasedRealTimeAggregateFeatureHydratorParam extends FSParam[Boolean]( name = "home_mixer_enable_topic_based_real_time_aggregate_feature_hydrator_param", default = true ) object EnableTopicCountryBasedRealTimeAggregateFeatureHydratorParam extends FSParam[Boolean]( name = "home_mixer_enable_topic_country_based_real_time_aggregate_feature_hydrator_param", default = true ) object EnableTopicEdgeAggregateFeatureHydratorParam extends FSParam[Boolean]( name = "home_mixer_enable_topic_edge_aggregate_feature_hydrator_param", default = true ) object FeedbackFatigueFilteringDurationParam extends FSBoundedParam[Duration]( name = "home_mixer_feedback_fatigue_filtering_duration_in_days", default = 14.days, min = 0.days, max = 100.days ) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromDays } object EnableCommonFeaturesDataRecordCopyDuringPldrConversionParam extends BooleanDeciderParam( decider = DeciderKey.EnableCommonFeaturesDataRecordCopyDuringPldrConversion) object EnablePinnedTweetsCarouselParam extends FSParam[Boolean]( name = "home_mixer_enable_pinned_tweets_carousel", default = false ) object EnablePostFeedbackParam extends FSParam[Boolean]( name = "home_mixer_enable_post_feedback", default = false ) object PostFeedbackThresholdParam extends FSBoundedParam[Double]( name = "home_mixer_post_feedback_threshold", default = 0.0, min = 0.0, max = 1.0 ) object PostFeedbackPromptTitleParam extends FSParam[String]( name = "home_mixer_post_feedback_prompt_title", default = "Are you interested in this post?" ) object PostFeedbackPromptPositiveParam extends FSParam[String]( name = "home_mixer_post_feedback_prompt_positive", default = "Yes" ) object PostFeedbackPromptNegativeParam extends FSParam[String]( name = "home_mixer_post_feedback_prompt_negative", default = "No" ) object PostFeedbackPromptNeutralParam extends FSParam[String]( name = "home_mixer_post_feedback_prompt_neutral", default = "Not sure" ) object EnablePostFollowupParam extends FSParam[Boolean]( name = "home_mixer_enable_post_followup", default = false ) object EnablePostDetailsNegativeFeedbackParam extends FSParam[Boolean]( name = "home_mixer_enable_post_details_negative_feedback", default = false ) object PostFollowupThresholdParam extends FSBoundedParam[Double]( name = "home_mixer_post_followup_threshold", default = 0.0, min = 0.0, max = 1.0 ) object EnableSlopFilter extends FSParam[Boolean]( name = "home_mixer_enable_slop_filter", default = false ) object EnableNsfwFilter extends FSParam[Boolean]( name = "home_mixer_enable_nsfw_filter", default = false ) object EnableSoftNsfwFilter extends FSParam[Boolean]( name = "home_mixer_enable_soft_nsfw_filter", default = false ) object EnableGrokSpamFilter extends FSParam[Boolean]( name = "home_mixer_enable_grok_spam_filter", default = false ) object EnableGrokViolentFilter extends FSParam[Boolean]( name = "home_mixer_enable_grok_violent_filter", default = false ) object EnableGrokGoreFilter extends FSParam[Boolean]( name = "home_mixer_enable_grok_gore_filter", default = false ) object EnableMinVideoDurationFilter extends FSParam[Boolean]( name = "home_mixer_enable_min_video_duration_filter", default = false ) object EnableMaxVideoDurationFilter extends FSParam[Boolean]( name = "home_mixer_enable_max_video_duration_filter", default = false ) object EnableClusterBasedDedupFilter extends FSParam[Boolean]( name = "home_mixer_enable_cluster_based_dedup_filter", default = false ) object EnableCountryFilter extends FSParam[Boolean]( name = "home_mixer_enable_country_filter", default = false ) object EnableRegionFilter extends FSParam[Boolean]( name = "home_mixer_enable_region_filter", default = false ) object EnableHasMultipleMediaFilter extends FSParam[Boolean]( name = "home_mixer_enable_has_multiple_media_filter", default = false ) object EnableClusterBased88DedupFilter extends FSParam[Boolean]( name = "home_mixer_enable_cluster_based_88_dedup_filter", default = false ) object EnableNoClusterFilter extends FSParam[Boolean]( name = "home_mixer_enable_no_cluster_filter", default = false ) object DedupHistoricalEventsTimeWindowParam extends FSBoundedParam[Long]( name = "home_mixer_dedup_historical_events_time_window", default = 43200000L, // 12 * 60 * 60 * 1000 = 12hrs in milliseconds min = 0L, max = 604800000L // 7 days ) object MinVideoDurationThresholdParam extends FSBoundedParam[Long]( name = "home_mixer_min_video_duration_threshold", default = 0L, // 0 second min = 0L, max = 604800000L // 7 days ) object MaxVideoDurationThresholdParam extends FSBoundedParam[Long]( name = "home_mixer_max_video_duration_threshold", default = 604800000L, // 7 days min = 0L, max = 604800000L // 7 days ) object EnableSlopFilterLowSignalUsers extends FSParam[Boolean]( name = "home_mixer_enable_slop_filter_low_signal_users", default = false ) object EnableSlopFilterEligibleUserStateParam extends FSParam[Boolean]( name = "home_mixer_enable_slop_filter_eligible_user_state_param", default = true ) object SlopMaxScore extends FSBoundedParam[Double]( name = "home_mixer_slop_max_score", default = 0.3, min = 0.0, max = 4.0 ) object SlopMinFollowers extends FSBoundedParam[Int]( name = "home_mixer_slop_min_followers", default = 100, min = 0, max = 10000000 ) object EnableGrokAnnotations extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_grok_annotations", default = false ) object UserActionsMaxCount extends FSBoundedParam[Int]( name = "home_mixer_user_actions_max_count", default = 522, min = 0, max = 10000 ) object EnableTweetRTAMhOnlyParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_tweet_rta_read_from_mh", default = false ) object EnableTweetRTAMhFallbackParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_tweet_rta_read_fallback_to_mh", default = false ) object EnableTweetCountryRTAMhOnlyParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_tweet_country_rta_read_from_mh", default = false ) object EnableTweetCountryRTAMhFallbackParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_tweet_country_rta_read_fallback_to_mh", default = false ) object EnableUserRTAMhOnlyParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_user_rta_read_from_mh", default = false ) object EnableUserRTAMhFallbackParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_user_rta_read_fallback_to_mh", default = false ) object EnableUserAuthorRTAMhOnlyParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_user_author_rta_read_from_mh", default = false ) object EnableUserAuthorRTAMhFallbackParam extends FSParam[Boolean]( name = "home_mixer_feature_hydration_enable_user_author_rta_read_fallback_to_mh", default = false ) object MaxPostContextPostsPerRequest extends FSParam[Int]( name = "home_mixer_feature_hydration_max_post_context_posts", default = 5 ) object MaxPostContextDuplicatesPerRequest extends FSParam[Int]( name = "home_mixer_feature_hydration_max_post_context_duplicates", default = 2 ) object PhoenixCluster extends Enumeration { val Prod = Value val Experiment1 = Value val Experiment2 = Value val Experiment3 = Value val Experiment4 = Value val Experiment5 = Value val Experiment6 = Value val Experiment7 = Value val Experiment8 = Value } object PhoenixInferenceClusterParam extends FSEnumParam[PhoenixCluster.type]( name = "home_mixer_model_phoenix_inference_cluster_id", default = PhoenixCluster.Prod, enum = PhoenixCluster ) object PhoenixTimeoutInMsParam extends FSBoundedParam[Int]( name = "home_mixer_model_phoenix_timeout_in_ms", default = 500, min = 10, max = 10000 ) object EnablePhoenixScorerParam extends FSParam[Boolean]( name = "home_mixer_model_enable_phoenix_scorer", default = false ) object EnableUserActionsShadowScribeParam extends FSParam[Boolean]( name = "home_mixer_enable_user_actions_shadow_scribe", default = false ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.scala ================================================ package com.twitter.home_mixer.param object HomeMixerFlagName { final val ScribeClientEventsFlag = "scribe.client_events" final val ScribeServedCandidatesFlag = "scribe.served_candidates" final val ScribeScoredCandidatesFlag = "scribe.scored_candidates" final val ScribeFeaturesFlag = "scribe.features" final val DataRecordMetadataStoreConfigsYmlFlag = "data.record.metadata.store.configs.yml" final val DarkTrafficFilterDeciderKey = "thrift.dark.traffic.filter.decider_key" final val TargetScoringLatency = "target.scoring.latency" } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.scala ================================================ package com.twitter.home_mixer.param object HomeMixerInjectionNames { final val AuthorFeatureRepository = "AuthorFeatureRepository" final val CommonFeaturesScribeEventPublisher = "CommonFeaturesScribeEventPublisher" final val CommonFeaturesScribeVideoEventPublisher = "CommonFeaturesScribeVideoEventPublisher" final val EarlybirdRepository = "EarlybirdRepository" final val EarlybirdRealtimCGEndpoint = "EarlybirdRealtimCGEndpoint" final val EngagementsReceivedByAuthorCache = "EngagementsReceivedByAuthorCache" final val EntityRealGraphClientStore = "EntityRealGraphClientStore" final val GraphTwoHopRepository = "GraphTwoHopRepository" final val GizmoduckTimelinesCache = "GizmoduckTimelinesCache" final val HomeAuthorFeaturesCacheClient = "HomeAuthorFeaturesCacheClient" final val InterestsThriftServiceClient = "InterestsThriftServiceClient" final val BatchedStratoClientWithDefaultTimeout = "BatchedStratoClientWithDefaultTimeout" final val StratoClientWithLongTimeout = "StratoClientWithLongTimeout" final val BatchedStratoClientWithModerateTimeout = "BatchedStratoClientWithModerateTimeout" final val BatchedStratoClientWithLongTimeout = "BatchedStratoClientWithLongTimeout" final val ManhattanApolloClient = "ManhattanApolloClient" final val ManhattanAthenaClient = "ManhattanAthenaClient" final val ManhattanOmegaClient = "ManhattanOmegaClient" final val ManhattanStarbuckClient = "ManhattanStarbuckClient" final val MediaClipClusterIdInMemCache = "MediaClipClusterIdInMemCache" final val ImageClipClusterIdInMemCache = "ImageClipClusterIdInMemCache" final val MediaCompletionRateInMemCache = "MediaCompletionRateInMemCache" final val IsColdStartPostInMemCache = "IsColdStartPostInMemCache" final val MemcacheCandidateFeaturesStore = "MemcacheCandidateFeaturesStore" final val MemcacheVideoCandidateFeaturesStore = "MemcacheVideoCandidateFeaturesStore" final val MemcachedImpressionBloomFilterStore = "MemcachedImpressionBloomFilterStore" final val NaviModelClientHomeRecap = "NaviModelClientHomeRecap" final val NaviModelClientHomeRecapSecondary = "NaviModelClientHomeRecapSecondary" final val NaviModelClientHomeRecapRealtime = "NaviModelClientHomeRecapRealtime" final val NaviModelClientHomeRecapGPU = "NaviModelClientHomeRecapGPU" final val NaviModelClientHomeRecapVideo = "NaviModelClientHomeRecapVideo" final val RealGraphInNetworkScoresOnPrem = "RealGraphInNetworkScoresOnPrem" final val RealGraphManhattanEndpoint = "RealGraphFeaturesManhattanEndpoint" final val RTAManhattanEndpoint = "RTAManhattanEndpoint" final val RTAManhattanStore = "RTAManhattanStore" final val RealGraphFeatureRepository = "RealGraphFeatureRepository" final val RealTimeInteractionGraphUserVertexCache = "RealTimeInteractionGraphUserVertexCache" final val RealTimeInteractionGraphUserVertexClient = "RealTimeInteractionGraphUserVertexClient" final val ScoredTweetsCache = "ScoredTweetsCache" final val ScoredVideoTweetsCache = "ScoredVideoTweetsCache" final val TesBatchedStratoClient = "TesBatchedStratoClient" final val TimelineAggregateMetadataRepository = "TimelineAggregateMetadataRepository" final val TimelineAggregatePartARepository = "TimelineAggregatePartARepository" final val TimelineAggregatePartBRepository = "TimelineAggregatePartBRepository" final val TimelinesRealTimeAggregateClient = "TimelinesRealTimeAggregateClient" final val TopicCountryEngagementCache = "TopicCountryEngagementCache" final val TopicEngagementCache = "TopicEngagementCache" final val TweetCountryEngagementCache = "TweetCountryEngagementCache" final val TweetEngagementCache = "TweetEngagementCache" final val TweetypieStaticEntitiesCache = "TweetypieStaticEntitiesCache" final val TwhinAuthorFollowFeatureCacheClient = "TwhinAuthorFollowFeatureCacheClient" final val TwhinAuthorFollowFeatureRepository = "TwhinAuthorFollowFeatureRepository" final val TwhinUserEngagementFeatureRepository = "TwhinUserEngagementFeatureRepository" final val TwhinRebuildUserEngagementFeatureRepository = "TwhinRebuildUserEngagementFeatureRepository" final val TwhinUserFollowFeatureRepository = "TwhinUserFollowFeatureRepository" final val TwhinTweetEmbeddingsStore = "TwhinTweetEmbeddingsStore" final val TwhinRebuildTweetEmbeddingsStore = "TwhinRebuildTweetEmbeddingsStore" final val TwhinVideoEmbeddingsStore = "TwhinVideoEmbeddingsStore" final val TwhinUserNegativeEmbeddingsStore = "TwhinUserNegativeEmbeddingsStore" final val TwhinUserPositiveEmbeddingsStore = "TwhinUserPositiveEmbeddingsStore" final val TwhinRebuildUserPositiveEmbeddingsStore = "TwhinRebuildUserPositiveEmbeddingsStore" final val TwitterListEngagementCache = "TwitterListEngagementCache" final val UserAuthorEngagementCache = "UserAuthorEngagementCache" final val UserEngagementCache = "UserEngagementCache" final val UserFollowedTopicIdsRepository = "UserFollowedTopicIdsRepository" final val UserLanguagesRepository = "UserLanguagesRepository" final val UserTopicEngagementForNewUserCache = "UserTopicEngagementForNewUserCache" final val UtegSocialProofRepository = "UtegSocialProofRepository" final val TvRealTimeAggregateClient = "TvRealTimeAggregateClient" final val TvVideoByUserTweetCache = "TvVideoByUserTweetCache" final val TvWatchedVideoIdsKeyCacheStore = "TvWatchedVideoIdsKeyCacheStore" final val MediaClusterId95Store = "MediaClusterId95Store" final val MediaClusterId88Store = "MediaClusterId88Store" final val TweetWatchTimeMetadataStore = "TweetWatchTimeMetadataStore" final val VideoEmbeddingMHStore = "VideoEmbeddingMHStore" } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = ["servo/decider"], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.scala ================================================ package com.twitter.home_mixer.param.decider import com.twitter.servo.decider.DeciderKeyEnum /** * These values must correspond to the deciders configured in the * home-mixer/server/src/main/resources/config/decider.yml file * * @see [[com.twitter.product_mixer.core.product.ProductParamConfig.enabledDeciderKey]] */ object DeciderKey extends DeciderKeyEnum { val EnableForYouProduct = Value("enable_for_you_product") val EnableFollowingProduct = Value("enable_following_product") val EnableScoredTweetsProduct = Value("enable_scored_tweets_product") val EnableScoredVideoTweetsProduct = Value("enable_scored_video_tweets_product") val EnableSubscribedProduct = Value("enable_subscribed_product") val EnableHeavyRankerScoresProduct = Value("enable_heavy_ranker_scores_product") val EnableSimClustersSimilarityFeatureHydration = Value("enable_simclusters_similarity_feature_hydration") val EnableServedCandidateFeatureKeysKafkaPublishing = Value("enable_served_candidate_feature_keys_kafka_publishing") val EnablePublishCommonFeaturesKafka = Value("enable_publish_common_features_kafka") val LiveSpacesFactor = Value("live_spaces_factor") val MtlNormalizationAlpha = Value("mtl_normalization_alpha") val EnableTweetypieContentFeatures = Value("enable_tweetypie_content_features") val EnableCommonFeaturesDataRecordCopyDuringPldrConversion = Value("enable_common_features_data_record_copy_during_pldr_conversion") val EnableGetTweetsFromArchiveIndex = Value("enable_get_tweets_from_archive_index") val EnableSecondaryNaviRecapCluster = Value("enable_secondary_navi_recap_cluster") val EnableGPUNaviRecapClusterTestUsers = Value("enable_gpu_navi_recap_cluster_test_users") val NaviGPUClusterRequestBatchSize = Value("navi_gpu_cluster_request_batch_size") val EnableVideoSummaryEmbeddingHydration = Value( "enable_video_summary_embedding_feature_hydration") val EnableVideoClipEmbeddingHydration = Value("enable_video_clip_embedding_feature_hydration") val EnableScoredVideoTweetsUserHistoryEventsQueryFeatureHydrationDeciderParam = Value( "enable_scored_video_tweets_user_history_events_query_feature_hydration") val EnableVideoClipEmbeddingMediaUnderstandingHydration = Value( "enable_video_clip_embedding_media_understanding_feature_hydration") } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.bazel ================================================ scala_library( compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/heavy_ranker_scores", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/heavy_ranker_scores/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_video_tweets", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model", "home-mixer/thrift/src/main/thrift:thrift-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeMixerProductModule.scala ================================================ package com.twitter.home_mixer.product import com.twitter.inject.TwitterModule import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistryConfig object HomeMixerProductModule extends TwitterModule { override def configure(): Unit = { bind[ProductPipelineRegistryConfig].to[HomeProductPipelineRegistryConfig] } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.scala ================================================ package com.twitter.home_mixer.product import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.HeavyRankerScoresProduct import com.twitter.home_mixer.model.request.ScoredTweetsProduct import com.twitter.home_mixer.model.request.ScoredVideoTweetsProduct import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.home_mixer.product.following.FollowingProductPipelineConfig import com.twitter.home_mixer.product.for_you.ForYouProductPipelineConfig import com.twitter.home_mixer.product.heavy_ranker_scores.HeavyRankerScoresProductPipelineConfig import com.twitter.home_mixer.product.scored_tweets.ScoredTweetsProductPipelineConfig import com.twitter.home_mixer.product.scored_video_tweets.ScoredVideoTweetsProductPipelineConfig import com.twitter.home_mixer.product.subscribed.SubscribedProductPipelineConfig import com.twitter.inject.Injector import com.twitter.product_mixer.core.product.guice.ProductScope import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistryConfig import javax.inject.Inject import javax.inject.Singleton @Singleton class HomeProductPipelineRegistryConfig @Inject() ( injector: Injector, productScope: ProductScope) extends ProductPipelineRegistryConfig { private val followingProductPipelineConfig = productScope.let(FollowingProduct) { injector.instance[FollowingProductPipelineConfig] } private val forYouProductPipelineConfig = productScope.let(ForYouProduct) { injector.instance[ForYouProductPipelineConfig] } private val scoredTweetsProductPipelineConfig = productScope.let(ScoredTweetsProduct) { injector.instance[ScoredTweetsProductPipelineConfig] } private val scoredVideoTweetsProductPipelineConfig = productScope.let(ScoredVideoTweetsProduct) { injector.instance[ScoredVideoTweetsProductPipelineConfig] } private val subscribedProductPipelineConfig = productScope.let(SubscribedProduct) { injector.instance[SubscribedProductPipelineConfig] } private val heavyRankerScoresProductPipelineConfig = productScope.let(HeavyRankerScoresProduct) { injector.instance[HeavyRankerScoresProductPipelineConfig] } override val productPipelineConfigs = Seq( followingProductPipelineConfig, forYouProductPipelineConfig, scoredTweetsProductPipelineConfig, scoredVideoTweetsProductPipelineConfig, subscribedProductPipelineConfig, heavyRankerScoresProductPipelineConfig ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "ads-injection/lib/src/main/scala/com/twitter/goldfinch/api", "home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/location", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline/selector", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/cursor/timelines", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/earlybird", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", "src/java/com/twitter/search/common/schema/base", "src/java/com/twitter/search/common/schema/earlybird", "src/java/com/twitter/search/queryparser/query:core-query-nodes", "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", "src/thrift/com/twitter/search:earlybird-scala", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", ], exports = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "src/thrift/com/twitter/timelines/render:thrift-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.scala ================================================ package com.twitter.home_mixer.product.following import com.twitter.adserver.{thriftscala => ads} import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder import com.twitter.home_mixer.functional_component.feature_hydrator.NoAdsTierFeature import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate import com.twitter.home_mixer.functional_component.gate.ExcludeSyntheticUserGate import com.twitter.home_mixer.param.HomeGlobalParams import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam import com.twitter.home_mixer.product.following.model.FollowingQuery import com.twitter.home_mixer.product.following.param.FollowingParam.EnableFastAds import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad.AdsCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator import com.twitter.product_mixer.component_library.gate.FeatureGate import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsCandidatePipelineConfigBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter import com.twitter.product_mixer.core.gate.ParamNotGate import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext import javax.inject.Inject import javax.inject.Singleton @Singleton class FollowingAdsCandidatePipelineBuilder @Inject() ( adsCandidatePipelineConfigBuilder: AdsCandidatePipelineConfigBuilder, adsCandidateSource: AdsProdThriftCandidateSource, advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ FollowingQuery, AdsCandidate ]) { private val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("FollowingAds") private val EstimatedNumOrganicItems = 100.toShort private val servedType = hmt.ServedType.FollowingPromoted private val clientEventInfoBuilder = ClientEventInfoBuilder( component = servedType.originalName, detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(servedType.originalName))) ) private val contextualTweetRefBuilder = ContextualTweetRefBuilder( TweetHydrationContext( safetyLevelOverride = Some(TimelineHomePromotedHydrationSafetyLevel), outerTweetContext = None )) private val decorator = UrtItemCandidateDecorator( AdsCandidateUrtItemBuilder( tweetClientEventInfoBuilder = Some(clientEventInfoBuilder), contextualTweetRefBuilder = Some(contextualTweetRefBuilder) )) private val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() ) def build(): AdsCandidatePipelineConfig[FollowingQuery] = adsCandidatePipelineConfigBuilder.build[FollowingQuery]( adsCandidateSource = adsCandidateSource, identifier = identifier, adsDisplayLocationBuilder = query => if (query.params.getBoolean(EnableFastAds)) ads.DisplayLocation.TimelineHomeReverseChron else ads.DisplayLocation.TimelineHome, estimateNumOrganicItems = _ => EstimatedNumOrganicItems, gates = Seq( ParamNotGate( name = "AdsDisableInjectionBasedOnUserRole", param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam ), ExcludeSoftUserGate, ExcludeSyntheticUserGate, FeatureGate.fromNegatedFeature(NoAdsTierFeature) ), filters = Seq(ValidAdImpressionIdFilter), postFilterFeatureHydration = Seq( ParamGatedCandidateFeatureHydrator( EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, advertiserBrandSafetySettingsFeatureHydrator ) ), decorator = Some(decorator), alerts = alerts, urtRequest = Some(true) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsDependentCandidatePipelineBuilder.scala ================================================ package com.twitter.home_mixer.product.following import com.twitter.adserver.{thriftscala => ads} import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder import com.twitter.home_mixer.functional_component.feature_hydrator.NoAdsTierFeature import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate import com.twitter.home_mixer.functional_component.gate.ExcludeSyntheticUserGate import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature import com.twitter.home_mixer.param.HomeGlobalParams import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam import com.twitter.home_mixer.product.following.model.FollowingQuery import com.twitter.home_mixer.product.following.param.FollowingParam.EnableFastAds import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad.AdsCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator import com.twitter.product_mixer.component_library.gate.FeatureGate import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfigBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.ads.CountCandidatesFromPipelines import com.twitter.product_mixer.component_library.pipeline.candidate.ads.PipelineScopedOrganicItems import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.gate.ParamNotGate import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext import javax.inject.Inject import javax.inject.Singleton @Singleton class FollowingAdsDependentCandidatePipelineBuilder @Inject() ( adsCandidatePipelineConfigBuilder: AdsDependentCandidatePipelineConfigBuilder, adsCandidateSource: AdsProdThriftCandidateSource, advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ FollowingQuery, AdsCandidate ]) { private val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("FollowingAdsDependent") private val servedType = hmt.ServedType.FollowingPromoted private val clientEventInfoBuilder = ClientEventInfoBuilder( component = servedType.originalName, detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(servedType.originalName))) ) private val contextualTweetRefBuilder = ContextualTweetRefBuilder( TweetHydrationContext( safetyLevelOverride = Some(TimelineHomePromotedHydrationSafetyLevel), outerTweetContext = None )) private val decorator = UrtItemCandidateDecorator( AdsCandidateUrtItemBuilder( tweetClientEventInfoBuilder = Some(clientEventInfoBuilder), contextualTweetRefBuilder = Some(contextualTweetRefBuilder) )) private val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() ) def build( organicCandidatePipelines: CandidateScope ): AdsDependentCandidatePipelineConfig[FollowingQuery] = adsCandidatePipelineConfigBuilder.build[FollowingQuery]( adsCandidateSource = adsCandidateSource, identifier = identifier, adsDisplayLocationBuilder = query => if (query.params.getBoolean(EnableFastAds)) ads.DisplayLocation.TimelineHomeReverseChron else ads.DisplayLocation.TimelineHome, getOrganicItems = PipelineScopedOrganicItems( pipelines = organicCandidatePipelines, textFeature = TweetTextFeature, languageFeature = TweetLanguageFeature ), countNumOrganicItems = CountCandidatesFromPipelines(organicCandidatePipelines), gates = Seq( ParamNotGate( name = "AdsDisableInjectionBasedOnUserRole", param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam ), ExcludeSoftUserGate, ExcludeSyntheticUserGate, FeatureGate.fromNegatedFeature(NoAdsTierFeature), NonEmptyCandidatesGate(organicCandidatePipelines) ), filters = Seq(ValidAdImpressionIdFilter), postFilterFeatureHydration = Seq( ParamGatedCandidateFeatureHydrator( EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, advertiserBrandSafetySettingsFeatureHydrator ) ), decorator = Some(decorator), alerts = alerts, urtRequest = Some(true), ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingDependentAdsMixerPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.following import com.twitter.clientapp.{thriftscala => ca} import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas import com.twitter.home_mixer.candidate_pipeline.ConversationServiceCandidatePipelineConfigBuilder import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig import com.twitter.home_mixer.candidate_pipeline.VerifiedPromptCandidatePipelineConfig import com.twitter.home_mixer.functional_component.decorator.urt.builder.AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder import com.twitter.home_mixer.functional_component.feature_hydrator._ import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration import com.twitter.home_mixer.functional_component.side_effect._ import com.twitter.home_mixer.model.ClearCacheIncludeInstruction import com.twitter.home_mixer.model.NavigationIncludeInstruction import com.twitter.home_mixer.param.HomeGlobalParams.EnableSSPAdsBrandSafetySettingsFeatureHydratorParam import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag import com.twitter.home_mixer.product.following.model.FollowingQuery import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.following.param.FollowingParam.ClearCache import com.twitter.home_mixer.product.following.param.FollowingParam.EnableFlipInjectionModuleCandidatePipelineParam import com.twitter.home_mixer.product.following.param.FollowingParam.EnablePostContextFeatureHydratorParam import com.twitter.home_mixer.product.following.param.FollowingParam.Navigation import com.twitter.home_mixer.product.following.param.FollowingParam.ServerMaxResultsParam import com.twitter.home_mixer.product.following.param.FollowingParam.StaticParamValueFive import com.twitter.home_mixer.product.following.param.FollowingParam.StaticParamValueZero import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.feature_hydrator.query.ads.SSPAdsBrandSafetySettingsFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.location.UserLocationQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptDependentCandidatePipelineConfigBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.selector.FlipPromptDynamicInsertionPosition import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.NavigationInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedCursorIdSelector.TweetIdSelector import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedGapCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowCoverInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.earlybird.EarlybirdGapIncludeInstruction import com.twitter.product_mixer.component_library.selector.DropMaxCandidates import com.twitter.product_mixer.component_library.selector.DropMaxModuleItemCandidates import com.twitter.product_mixer.component_library.selector.DropModuleTooFewModuleItemResults import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.component_library.selector.InsertDynamicPositionResults import com.twitter.product_mixer.component_library.selector.InsertFixedPositionResults import com.twitter.product_mixer.component_library.selector.SelectConditionally import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates import com.twitter.product_mixer.component_library.selector.ads.AdsInjector import com.twitter.product_mixer.component_library.selector.ads.InsertAdResults import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines import com.twitter.product_mixer.core.functional_component.configapi.StaticParam import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig import com.twitter.product_mixer.core.pipeline.FailOpenPolicy import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.render.{thriftscala => urt} import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton class FollowingDependentAdsMixerPipelineConfig @Inject() ( followingEarlybirdCandidatePipelineConfig: FollowingEarlybirdCandidatePipelineConfig, conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[ FollowingQuery ], followingAdsDependentCandidatePipelineBuilder: FollowingAdsDependentCandidatePipelineBuilder, followingWhoToFollowCandidatePipelineConfigBuilder: FollowingWhoToFollowCandidatePipelineConfigBuilder, flipPromptDependentCandidatePipelineConfigBuilder: FlipPromptDependentCandidatePipelineConfigBuilder, editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[FollowingQuery], verifiedPromptCandidatePipelineConfig: VerifiedPromptCandidatePipelineConfig, dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, rateLimitQueryFeatureHydrator: RateLimitQueryFeatureHydrator, requestQueryFeatureHydrator: RequestQueryFeatureHydrator[FollowingQuery], sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator, userLocationQueryFeatureHydrator: UserLocationQueryFeatureHydrator, userSubscriptionQueryFeatureHydrator: UserSubscriptionQueryFeatureHydrator, sspAdsBrandSafetySettingsFeatureHydrator: SSPAdsBrandSafetySettingsFeatureHydrator, adsInjector: AdsInjector, updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[FollowingQuery, Timeline], publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, homeTimelineServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter], urtTransportMarshaller: UrtTransportMarshaller, @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) extends MixerPipelineConfig[FollowingQuery, Timeline, urt.TimelineResponse] { override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("FollowingDependentAds") private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep private val resultSelectorsStep = MixerPipelineConfig.resultSelectorsStep override val fetchQueryFeatures: Seq[QueryFeatureHydrator[FollowingQuery]] = Seq( requestQueryFeatureHydrator, sgsFollowedUsersQueryFeatureHydrator, rateLimitQueryFeatureHydrator, gizmoduckUserQueryFeatureHydrator, userSubscriptionQueryFeatureHydrator, ParamGatedQueryFeatureHydrator( EnableSSPAdsBrandSafetySettingsFeatureHydratorParam, sspAdsBrandSafetySettingsFeatureHydrator ), AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, persistenceStoreQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, lastNonPollingTimeQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, userLocationQueryFeatureHydrator), AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator), ) private val earlybirdCandidatePipelineScope = SpecificPipeline(followingEarlybirdCandidatePipelineConfig.identifier) private val conversationServiceCandidatePipelineConfig = conversationServiceCandidatePipelineConfigBuilder.build( earlybirdCandidatePipelineScope, hmt.ServedType.FollowingInNetwork, EnablePostContextFeatureHydratorParam ) private val followingAdsCandidatePipelineConfig = followingAdsDependentCandidatePipelineBuilder.build(earlybirdCandidatePipelineScope) private val followingWhoToFollowCandidatePipelineConfig = followingWhoToFollowCandidatePipelineConfigBuilder.build(earlybirdCandidatePipelineScope) private val flipPromptCandidatePipelineConfig = flipPromptDependentCandidatePipelineConfigBuilder.build[FollowingQuery]( supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) ) override val candidatePipelines: Seq[CandidatePipelineConfig[FollowingQuery, _, _, _]] = Seq(followingEarlybirdCandidatePipelineConfig) override val dependentCandidatePipelines: Seq[ DependentCandidatePipelineConfig[FollowingQuery, _, _, _] ] = Seq( conversationServiceCandidatePipelineConfig, followingAdsCandidatePipelineConfig, followingWhoToFollowCandidatePipelineConfig, flipPromptCandidatePipelineConfig, editedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig, verifiedPromptCandidatePipelineConfig ) override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( followingAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, followingWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, ) override val resultSelectors: Seq[Selector[FollowingQuery]] = Seq( UpdateSortCandidates( ordering = CandidatesUtil.reverseChronTweetsOrdering, candidatePipeline = conversationServiceCandidatePipelineConfig.identifier ), DropMaxCandidates( candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, maxSelectionsParam = MaxNumberReplaceInstructionsParam ), DropMaxCandidates( candidatePipeline = conversationServiceCandidatePipelineConfig.identifier, maxSelectionsParam = ServerMaxResultsParam ), DropModuleTooFewModuleItemResults( candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, minModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MinCandidatesSize) ), DropMaxModuleItemCandidates( candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, maxModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MaxCandidatesSize) ), InsertAppendResults(candidatePipeline = conversationServiceCandidatePipelineConfig.identifier), InsertFixedPositionResults( candidatePipeline = verifiedPromptCandidatePipelineConfig.identifier, positionParam = StaticParamValueZero ), InsertDynamicPositionResults( candidatePipeline = flipPromptCandidatePipelineConfig.identifier, dynamicInsertionPosition = FlipPromptDynamicInsertionPosition(StaticParamValueZero) ), InsertFixedPositionResults( candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, positionParam = StaticParamValueFive ), InsertAdResults( surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), adsCandidatePipeline = followingAdsCandidatePipelineConfig.identifier ), // This selector must come after the tweets are inserted into the results UpdateNewTweetsPillDecoration( pipelineScope = SpecificPipelines( conversationServiceCandidatePipelineConfig.identifier, newTweetsPillCandidatePipelineConfig.identifier ), stringCenter = stringCenterProvider.get(), seeNewTweetsString = externalStrings.seeNewTweetsString, tweetedString = externalStrings.tweetedString ), InsertAppendResults(candidatePipeline = editedTweetsCandidatePipelineConfig.identifier), SelectConditionally( selector = InsertAppendResults(candidatePipeline = newTweetsPillCandidatePipelineConfig.identifier), includeSelector = (_, _, results) => CandidatesUtil.containsType[TweetCandidate](results) ), UpdateHomeClientEventDetails( candidatePipelines = Set(conversationServiceCandidatePipelineConfig.identifier) ) ) private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( enableScribeClientEvents = enableScribeClientEvents, logPipelinePublisher = clientEventsScribeEventPublisher, injectedTweetsCandidatePipelineIdentifiers = Seq(conversationServiceCandidatePipelineConfig.identifier), adsCandidatePipelineIdentifier = Some(followingAdsCandidatePipelineConfig.identifier), whoToFollowCandidatePipelineIdentifier = Some(followingWhoToFollowCandidatePipelineConfig.identifier), ) override val resultSideEffects: Seq[PipelineResultSideEffect[FollowingQuery, Timeline]] = Seq( homeScribeClientEventSideEffect, homeTimelineServedCandidatesSideEffect, publishClientSentImpressionsEventBusSideEffect, truncateTimelinesPersistenceStoreSideEffect, updateLastNonPollingTimeSideEffect, updateTimelinesPersistenceStoreSideEffect, ) override val domainMarshaller: DomainMarshaller[FollowingQuery, Timeline] = { val instructionBuilders = Seq( ClearCacheInstructionBuilder( ClearCacheIncludeInstruction( ClearCache.PtrEnableParam, ClearCache.ColdStartEnableParam, ClearCache.WarmStartEnableParam, ClearCache.ManualRefreshEnableParam, ClearCache.NavigateEnableParam, ClearCache.MinEntriesParam )), ReplaceEntryInstructionBuilder(ReplaceAllEntries), AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder(), ShowAlertInstructionBuilder(), ShowCoverInstructionBuilder(), NavigationInstructionBuilder( NavigationIncludeInstruction( Navigation.PtrEnableParam, Navigation.ColdStartEnableParam, Navigation.WarmStartEnableParam, Navigation.ManualRefreshEnableParam, Navigation.NavigateEnableParam )) ) val topCursorBuilder = OrderedTopCursorBuilder(TweetIdSelector) val bottomCursorBuilder = OrderedBottomCursorBuilder(TweetIdSelector, EarlybirdGapIncludeInstruction.inverse()) val gapCursorBuilder = OrderedGapCursorBuilder(TweetIdSelector, EarlybirdGapIncludeInstruction) val scribeConfigBuilder = StaticTimelineScribeConfigBuilder(TimelineScribeConfig(Some("following"), None, None)) val metadataBuilder = UrtMetadataBuilder(scribeConfigBuilder = Some(scribeConfigBuilder)) UrtDomainMarshaller( instructionBuilders = instructionBuilders, metadataBuilder = Some(metadataBuilder), cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder, gapCursorBuilder) ) } override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = urtTransportMarshaller } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.following import com.twitter.home_mixer.candidate_pipeline.FollowingEarlybirdResponseFeatureTransformer import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.product.following.model.FollowingQuery import com.twitter.product_mixer.component_library.candidate_source.earlybird.EarlybirdTweetCandidateSource import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.search.earlybird.{thriftscala => t} import javax.inject.Inject import javax.inject.Singleton @Singleton class FollowingEarlybirdCandidatePipelineConfig @Inject() ( earlybirdTweetCandidateSource: EarlybirdTweetCandidateSource, followingEarlybirdQueryTransformer: FollowingEarlybirdQueryTransformer) extends CandidatePipelineConfig[ FollowingQuery, t.EarlybirdRequest, t.ThriftSearchResult, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("FollowingEarlybird") override val candidateSource: BaseCandidateSource[t.EarlybirdRequest, t.ThriftSearchResult] = earlybirdTweetCandidateSource override val gates: Seq[Gate[FollowingQuery]] = Seq( RateLimitGate, NonEmptySeqFeatureGate(SGSFollowedUsersFeature) ) override val queryTransformer: CandidatePipelineQueryTransformer[ FollowingQuery, t.EarlybirdRequest ] = followingEarlybirdQueryTransformer override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[t.ThriftSearchResult] ] = Seq(FollowingEarlybirdResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ t.ThriftSearchResult, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.id) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.scala ================================================ package com.twitter.home_mixer.product.following import com.twitter.finagle.thrift.ClientId import com.twitter.finagle.tracing.Trace import com.twitter.home_mixer.product.following.model.FollowingQuery import com.twitter.home_mixer.product.following.param.FollowingParam.ServerMaxResultsParam import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant import com.twitter.search.earlybird.{thriftscala => t} import com.twitter.search.queryparser.query.Conjunction import com.twitter.search.queryparser.query.search.SearchOperator import javax.inject.Inject import javax.inject.Singleton import scala.jdk.CollectionConverters.asJavaIterableConverter @Singleton case class FollowingEarlybirdQueryTransformer @Inject() (clientId: ClientId) extends CandidatePipelineQueryTransformer[FollowingQuery, t.EarlybirdRequest] { override def transform(query: FollowingQuery): t.EarlybirdRequest = { val followedUserIds = query.features.map(_.get(SGSFollowedUsersFeature)).getOrElse(Seq.empty).toSet val userId = query.getRequiredUserId val combinedUserIds = userId +: followedUserIds.toSeq val baseFollowedUsersSearchOperator = new SearchOperator.Builder() .setType(SearchOperator.Type.FEATURE_VALUE_IN_ACCEPT_LIST_OR_UNSET) .addOperand(EarlybirdFieldConstant.DIRECTED_AT_USER_ID_CSF.getFieldName) val followedUsersQuery = baseFollowedUsersSearchOperator.addOperands(combinedUserIds.map(_.toString).asJava).build() val searchQuery = query.pipelineCursor .map { cursor => val sinceIdQuery = (id: Long) => new SearchOperator(SearchOperator.Type.SINCE_ID, id.toString) val maxIdQuery = // max ID is inclusive, so subtract 1 (id: Long) => new SearchOperator(SearchOperator.Type.MAX_ID, (id - 1).toString) (cursor.cursorType, cursor.id, cursor.gapBoundaryId) match { case (Some(TopCursor), Some(sinceId), _) => new Conjunction(sinceIdQuery(sinceId), followedUsersQuery) case (Some(BottomCursor), Some(maxId), _) => new Conjunction(maxIdQuery(maxId), followedUsersQuery) case (Some(GapCursor), Some(maxId), Some(sinceId)) => new Conjunction(sinceIdQuery(sinceId), maxIdQuery(maxId), followedUsersQuery) case (Some(GapCursor), _, _) => throw PipelineFailure(MalformedCursor, "Invalid cursor " + cursor.toString) case _ => followedUsersQuery } }.getOrElse(followedUsersQuery) val metadataOptions = t.ThriftSearchResultMetadataOptions( getInReplyToStatusId = true, getReferencedTweetAuthorId = true, getFromUserId = true ) t.EarlybirdRequest( searchQuery = t.ThriftSearchQuery( serializedQuery = Some(searchQuery.serialize), fromUserIDFilter64 = Some(combinedUserIds), numResults = query.requestedMaxResults.getOrElse(query.params(ServerMaxResultsParam)), rankingMode = t.ThriftSearchRankingMode.Recency, resultMetadataOptions = Some(metadataOptions), searcherId = query.getOptionalUserId, ), getOlderResults = Some(true), // needed for archive access to older tweets clientRequestID = Some(s"${Trace.id.traceId}"), followedUserIds = Some(combinedUserIds), numResultsToReturnAtRoot = Some(query.params(ServerMaxResultsParam)), clientId = Some(clientId.name), ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.candidate_pipeline import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.search.earlybird.{thriftscala => t} object FollowingEarlybirdResponseFeatureTransformer extends CandidateFeatureTransformer[t.ThriftSearchResult] { override val identifier: TransformerIdentifier = TransformerIdentifier("FollowingEarlybirdResponse") override val features: Set[Feature[_, _]] = Set( AuthorIdFeature, InReplyToTweetIdFeature, IsRetweetFeature, SourceTweetIdFeature, SourceUserIdFeature, ) override def transform(candidate: t.ThriftSearchResult): FeatureMap = FeatureMapBuilder() .add(AuthorIdFeature, candidate.tweetypieTweet.flatMap(_.coreData.map(_.userId))) .add( InReplyToTweetIdFeature, candidate.tweetypieTweet.flatMap(_.coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId)))) .add(IsRetweetFeature, candidate.metadata.exists(_.isRetweet.contains(true))) .add(SourceTweetIdFeature, candidate.sourceTweetypieTweet.map(_.id)) .add(SourceUserIdFeature, candidate.sourceTweetypieTweet.flatMap(_.coreData.map(_.userId))) .build() } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.following import com.twitter.clientapp.{thriftscala => ca} import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas import com.twitter.home_mixer.candidate_pipeline.ConversationServiceCandidatePipelineConfigBuilder import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig import com.twitter.home_mixer.candidate_pipeline.VerifiedPromptCandidatePipelineConfig import com.twitter.home_mixer.functional_component.decorator.urt.builder.AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder import com.twitter.home_mixer.functional_component.feature_hydrator._ import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration import com.twitter.home_mixer.functional_component.side_effect._ import com.twitter.home_mixer.model.ClearCacheIncludeInstruction import com.twitter.home_mixer.model.NavigationIncludeInstruction import com.twitter.home_mixer.param.HomeGlobalParams.EnableSSPAdsBrandSafetySettingsFeatureHydratorParam import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag import com.twitter.home_mixer.product.following.model.FollowingQuery import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.following.param.FollowingParam.ClearCache import com.twitter.home_mixer.product.following.param.FollowingParam.EnableFlipInjectionModuleCandidatePipelineParam import com.twitter.home_mixer.product.following.param.FollowingParam.EnablePostContextFeatureHydratorParam import com.twitter.home_mixer.product.following.param.FollowingParam.Navigation import com.twitter.home_mixer.product.following.param.FollowingParam.ServerMaxResultsParam import com.twitter.home_mixer.product.following.param.FollowingParam.StaticParamValueFive import com.twitter.home_mixer.product.following.param.FollowingParam.StaticParamValueZero import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.feature_hydrator.query.ads.SSPAdsBrandSafetySettingsFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.location.UserLocationQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptDependentCandidatePipelineConfigBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.selector.FlipPromptDynamicInsertionPosition import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.NavigationInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedCursorIdSelector.TweetIdSelector import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedGapCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowCoverInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.earlybird.EarlybirdGapIncludeInstruction import com.twitter.product_mixer.component_library.selector.DropMaxCandidates import com.twitter.product_mixer.component_library.selector.DropMaxModuleItemCandidates import com.twitter.product_mixer.component_library.selector.DropModuleTooFewModuleItemResults import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.component_library.selector.InsertDynamicPositionResults import com.twitter.product_mixer.component_library.selector.InsertFixedPositionResults import com.twitter.product_mixer.component_library.selector.SelectConditionally import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates import com.twitter.product_mixer.component_library.selector.ads.AdsInjector import com.twitter.product_mixer.component_library.selector.ads.InsertAdResults import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines import com.twitter.product_mixer.core.functional_component.configapi.StaticParam import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig import com.twitter.product_mixer.core.pipeline.FailOpenPolicy import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.render.{thriftscala => urt} import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton class FollowingMixerPipelineConfig @Inject() ( followingEarlybirdCandidatePipelineConfig: FollowingEarlybirdCandidatePipelineConfig, conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[ FollowingQuery ], followingAdsCandidatePipelineBuilder: FollowingAdsCandidatePipelineBuilder, followingWhoToFollowCandidatePipelineConfigBuilder: FollowingWhoToFollowCandidatePipelineConfigBuilder, flipPromptDependentCandidatePipelineConfigBuilder: FlipPromptDependentCandidatePipelineConfigBuilder, editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[FollowingQuery], verifiedPromptCandidatePipelineConfig: VerifiedPromptCandidatePipelineConfig, dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, rateLimitQueryFeatureHydrator: RateLimitQueryFeatureHydrator, requestQueryFeatureHydrator: RequestQueryFeatureHydrator[FollowingQuery], sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator, userLocationQueryFeatureHydrator: UserLocationQueryFeatureHydrator, userSubscriptionQueryFeatureHydrator: UserSubscriptionQueryFeatureHydrator, sspAdsBrandSafetySettingsFeatureHydrator: SSPAdsBrandSafetySettingsFeatureHydrator, adsInjector: AdsInjector, updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[FollowingQuery, Timeline], publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, homeTimelineServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter], urtTransportMarshaller: UrtTransportMarshaller, @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) extends MixerPipelineConfig[FollowingQuery, Timeline, urt.TimelineResponse] { override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("Following") private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep private val resultSelectorsStep = MixerPipelineConfig.resultSelectorsStep override val fetchQueryFeatures: Seq[QueryFeatureHydrator[FollowingQuery]] = Seq( requestQueryFeatureHydrator, sgsFollowedUsersQueryFeatureHydrator, rateLimitQueryFeatureHydrator, gizmoduckUserQueryFeatureHydrator, userSubscriptionQueryFeatureHydrator, ParamGatedQueryFeatureHydrator( EnableSSPAdsBrandSafetySettingsFeatureHydratorParam, sspAdsBrandSafetySettingsFeatureHydrator ), AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, persistenceStoreQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, lastNonPollingTimeQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, userLocationQueryFeatureHydrator), AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator), ) private val earlybirdCandidatePipelineScope = SpecificPipeline(followingEarlybirdCandidatePipelineConfig.identifier) private val conversationServiceCandidatePipelineConfig = conversationServiceCandidatePipelineConfigBuilder.build( earlybirdCandidatePipelineScope, hmt.ServedType.FollowingInNetwork, EnablePostContextFeatureHydratorParam ) private val followingAdsCandidatePipelineConfig = followingAdsCandidatePipelineBuilder.build() private val followingWhoToFollowCandidatePipelineConfig = followingWhoToFollowCandidatePipelineConfigBuilder.build(earlybirdCandidatePipelineScope) private val flipPromptCandidatePipelineConfig = flipPromptDependentCandidatePipelineConfigBuilder.build[FollowingQuery]( supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) ) override val candidatePipelines: Seq[CandidatePipelineConfig[FollowingQuery, _, _, _]] = Seq( followingEarlybirdCandidatePipelineConfig, followingAdsCandidatePipelineConfig ) override val dependentCandidatePipelines: Seq[ DependentCandidatePipelineConfig[FollowingQuery, _, _, _] ] = Seq( conversationServiceCandidatePipelineConfig, followingWhoToFollowCandidatePipelineConfig, flipPromptCandidatePipelineConfig, editedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig, verifiedPromptCandidatePipelineConfig ) override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( followingAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, followingWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, ) override val resultSelectors: Seq[Selector[FollowingQuery]] = Seq( UpdateSortCandidates( ordering = CandidatesUtil.reverseChronTweetsOrdering, candidatePipeline = conversationServiceCandidatePipelineConfig.identifier ), DropMaxCandidates( candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, maxSelectionsParam = MaxNumberReplaceInstructionsParam ), DropMaxCandidates( candidatePipeline = conversationServiceCandidatePipelineConfig.identifier, maxSelectionsParam = ServerMaxResultsParam ), DropModuleTooFewModuleItemResults( candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, minModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MinCandidatesSize) ), DropMaxModuleItemCandidates( candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, maxModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MaxCandidatesSize) ), InsertAppendResults(candidatePipeline = conversationServiceCandidatePipelineConfig.identifier), InsertFixedPositionResults( candidatePipeline = verifiedPromptCandidatePipelineConfig.identifier, positionParam = StaticParamValueZero ), InsertDynamicPositionResults( candidatePipeline = flipPromptCandidatePipelineConfig.identifier, dynamicInsertionPosition = FlipPromptDynamicInsertionPosition(StaticParamValueZero) ), InsertFixedPositionResults( candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, positionParam = StaticParamValueFive ), InsertAdResults( surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), adsCandidatePipeline = followingAdsCandidatePipelineConfig.identifier ), // This selector must come after the tweets are inserted into the results UpdateNewTweetsPillDecoration( pipelineScope = SpecificPipelines( conversationServiceCandidatePipelineConfig.identifier, newTweetsPillCandidatePipelineConfig.identifier ), stringCenter = stringCenterProvider.get(), seeNewTweetsString = externalStrings.seeNewTweetsString, tweetedString = externalStrings.tweetedString ), InsertAppendResults(candidatePipeline = editedTweetsCandidatePipelineConfig.identifier), SelectConditionally( selector = InsertAppendResults(candidatePipeline = newTweetsPillCandidatePipelineConfig.identifier), includeSelector = (_, _, results) => CandidatesUtil.containsType[TweetCandidate](results) ), UpdateHomeClientEventDetails( candidatePipelines = Set(conversationServiceCandidatePipelineConfig.identifier) ) ) private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( enableScribeClientEvents = enableScribeClientEvents, logPipelinePublisher = clientEventsScribeEventPublisher, injectedTweetsCandidatePipelineIdentifiers = Seq(conversationServiceCandidatePipelineConfig.identifier), adsCandidatePipelineIdentifier = Some(followingAdsCandidatePipelineConfig.identifier), whoToFollowCandidatePipelineIdentifier = Some(followingWhoToFollowCandidatePipelineConfig.identifier), ) override val resultSideEffects: Seq[PipelineResultSideEffect[FollowingQuery, Timeline]] = Seq( homeScribeClientEventSideEffect, homeTimelineServedCandidatesSideEffect, publishClientSentImpressionsEventBusSideEffect, truncateTimelinesPersistenceStoreSideEffect, updateLastNonPollingTimeSideEffect, updateTimelinesPersistenceStoreSideEffect, ) override val domainMarshaller: DomainMarshaller[FollowingQuery, Timeline] = { val instructionBuilders = Seq( ClearCacheInstructionBuilder( ClearCacheIncludeInstruction( ClearCache.PtrEnableParam, ClearCache.ColdStartEnableParam, ClearCache.WarmStartEnableParam, ClearCache.ManualRefreshEnableParam, ClearCache.NavigateEnableParam, ClearCache.MinEntriesParam )), ReplaceEntryInstructionBuilder(ReplaceAllEntries), AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder(), ShowAlertInstructionBuilder(), ShowCoverInstructionBuilder(), NavigationInstructionBuilder( NavigationIncludeInstruction( Navigation.PtrEnableParam, Navigation.ColdStartEnableParam, Navigation.WarmStartEnableParam, Navigation.ManualRefreshEnableParam, Navigation.NavigateEnableParam )) ) val topCursorBuilder = OrderedTopCursorBuilder(TweetIdSelector) val bottomCursorBuilder = OrderedBottomCursorBuilder(TweetIdSelector, EarlybirdGapIncludeInstruction.inverse()) val gapCursorBuilder = OrderedGapCursorBuilder(TweetIdSelector, EarlybirdGapIncludeInstruction) val scribeConfigBuilder = StaticTimelineScribeConfigBuilder(TimelineScribeConfig(Some("following"), None, None)) val metadataBuilder = UrtMetadataBuilder(scribeConfigBuilder = Some(scribeConfigBuilder)) UrtDomainMarshaller( instructionBuilders = instructionBuilders, metadataBuilder = Some(metadataBuilder), cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder, gapCursorBuilder) ) } override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = urtTransportMarshaller } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingProductPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.following import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.FollowingProductContext import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.product.following.model.FollowingQuery import com.twitter.home_mixer.product.following.param.FollowingParam.EnableDependentAdsParam import com.twitter.home_mixer.product.following.param.FollowingParam.ServerMaxResultsParam import com.twitter.home_mixer.product.following.param.FollowingParamConfig import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer import com.twitter.product_mixer.component_library.premarshaller.cursor.timelines.ChronologicalCursorUnmarshaller import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy import com.twitter.product_mixer.core.functional_component.common.alert.Alert import com.twitter.product_mixer.core.functional_component.common.alert.EmptyResponseRateAlert import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert import com.twitter.product_mixer.core.functional_component.common.alert.P99 import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert import com.twitter.product_mixer.core.functional_component.common.alert.ThroughputAlert import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.request.Product import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor import com.twitter.product_mixer.core.pipeline.PipelineConfig import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig import com.twitter.product_mixer.core.product.ProductParamConfig import com.twitter.product_mixer.core.util.SortIndexBuilder import com.twitter.timelines.configapi.Params import com.twitter.timelines.render.{thriftscala => urt} import com.twitter.timelines.util.RequestCursorSerializer import com.twitter.util.Time import com.twitter.util.Try import javax.inject.Inject import javax.inject.Singleton @Singleton class FollowingProductPipelineConfig @Inject() ( followingDependentAdsMixerPipelineConfig: FollowingDependentAdsMixerPipelineConfig, followingMixerPipelineConfig: FollowingMixerPipelineConfig, followingParamConfig: FollowingParamConfig) extends ProductPipelineConfig[HomeMixerRequest, FollowingQuery, urt.TimelineResponse] { override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("Following") override val product: Product = FollowingProduct override val paramConfig: ProductParamConfig = followingParamConfig override def pipelineQueryTransformer( request: HomeMixerRequest, params: Params ): FollowingQuery = { val context = request.productContext match { case Some(context: FollowingProductContext) => context case _ => throw PipelineFailure(BadRequest, "FollowingProductContext not found") } val debugOptions = request.debugParams.flatMap(_.debugOptions) /** * Unlike other clients, newly created tweets on Android have the sort index set to the current * time instead of the top sort index + 1, so these tweets get stuck at the top of the timeline * if subsequent timeline responses use the sort index from the previous response instead of * the current time. */ val pipelineCursor = request.serializedRequestCursor.flatMap { cursor => Try(UrtCursorSerializer.deserializeOrderedCursor(cursor)) .getOrElse(ChronologicalCursorUnmarshaller(RequestCursorSerializer.deserialize(cursor))) .map { case UrtOrderedCursor(_, id, Some(GapCursor), gapBoundaryId) if id.isEmpty || gapBoundaryId.isEmpty => throw PipelineFailure(MalformedCursor, "Gap Cursor bounds not defined") case topCursor @ UrtOrderedCursor(_, _, Some(TopCursor), _) => val queryTime = debugOptions.flatMap(_.requestTimeOverride).getOrElse(Time.now) topCursor.copy(initialSortIndex = SortIndexBuilder.timeToId(queryTime)) case cursor => cursor } } FollowingQuery( params = params, clientContext = request.clientContext, features = None, pipelineCursor = pipelineCursor, requestedMaxResults = Some(params(ServerMaxResultsParam)), debugOptions = debugOptions, deviceContext = context.deviceContext, seenTweetIds = context.seenTweetIds, dspClientContext = context.dspClientContext ) } override val pipelines: Seq[PipelineConfig] = Seq(followingMixerPipelineConfig, followingDependentAdsMixerPipelineConfig) override def pipelineSelector(query: FollowingQuery): ComponentIdentifier = if (query.params(EnableDependentAdsParam)) followingDependentAdsMixerPipelineConfig.identifier else followingMixerPipelineConfig.identifier override val alerts: Seq[Alert] = Seq( SuccessRateAlert( notificationGroup = DefaultNotificationGroup, warnPredicate = TriggerIfBelow(99.9, 20, 30), criticalPredicate = TriggerIfBelow(99.9, 30, 30), ), LatencyAlert( notificationGroup = DefaultNotificationGroup, percentile = P99, warnPredicate = TriggerIfLatencyAbove(1200.millis, 15, 30), criticalPredicate = TriggerIfLatencyAbove(1500.millis, 15, 30) ), ThroughputAlert( notificationGroup = DefaultNotificationGroup, warnPredicate = TriggerIfAbove(18000), criticalPredicate = TriggerIfAbove(20000) ), EmptyResponseRateAlert( notificationGroup = DefaultNotificationGroup, warnPredicate = TriggerIfAbove(65), criticalPredicate = TriggerIfAbove(80) ) ) override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.scala ================================================ package com.twitter.home_mixer.product.following import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeWhoToFollowFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature import com.twitter.home_mixer.product.following.model.FollowingQuery import com.twitter.home_mixer.product.following.param.FollowingParam.EnableWhoToFollowCandidatePipelineParam import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowDisplayLocationParam import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowDisplayTypeIdParam import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowMinInjectionIntervalParam import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowUserDisplayTypeIdParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ParamWhoToFollowModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.ParamWhoToFollowUserDisplayTypeBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmDependentCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmDependentCandidatePipelineConfigBuilder import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.functional_component.configapi.StaticParam import com.twitter.product_mixer.core.functional_component.gate.BaseGate import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelineservice.model.rich.EntityIdType import com.twitter.timelineservice.suggests.thriftscala.SuggestType import javax.inject.Inject import javax.inject.Singleton @Singleton class FollowingWhoToFollowCandidatePipelineConfigBuilder @Inject() ( whoToFollowArmDependentCandidatePipelineConfigBuilder: WhoToFollowArmDependentCandidatePipelineConfigBuilder, homeWhoToFollowFeedbackActionInfoBuilder: HomeWhoToFollowFeedbackActionInfoBuilder) { def build( requiredNonEmptyPipelines: CandidateScope ): WhoToFollowArmDependentCandidatePipelineConfig[FollowingQuery] = { val gates: Seq[BaseGate[PipelineQuery]] = Seq( TimelinesPersistenceStoreLastInjectionGate( WhoToFollowMinInjectionIntervalParam, EntityIdType.WhoToFollow ), DismissFatigueGate(SuggestType.WhoToFollow, DismissInfoFeature), NonEmptyCandidatesGate(requiredNonEmptyPipelines) ) whoToFollowArmDependentCandidatePipelineConfigBuilder.build[FollowingQuery]( identifier = WhoToFollowArmCandidatePipelineConfig.identifier, supportedClientParam = Some(EnableWhoToFollowCandidatePipelineParam), alerts = alerts, gates = gates, moduleDisplayTypeBuilder = ParamWhoToFollowModuleDisplayTypeBuilder(WhoToFollowDisplayTypeIdParam), feedbackActionInfoBuilder = Some(homeWhoToFollowFeedbackActionInfoBuilder), userDisplayTypeBuilder = ParamWhoToFollowUserDisplayTypeBuilder(WhoToFollowUserDisplayTypeIdParam), displayLocationParam = StaticParam(WhoToFollowDisplayLocationParam.default), excludedUserIdsFeature = Some(WhoToFollowExcludedUserIdsFeature), profileUserIdFeature = None ) } private val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(70), HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", "stringcenter/client", ], exports = [ "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/FollowingQuery.scala ================================================ package com.twitter.home_mixer.product.following.model import com.twitter.adserver.thriftscala.HomeTimelineType import com.twitter.adserver.thriftscala.TimelineRequestParams import com.twitter.home_mixer.model.HomeAdsQuery import com.twitter.dspbidder.commons.{thriftscala => dsp} import com.twitter.home_mixer.model.request.DeviceContext import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.model.request.HasSeenTweetIds import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.onboarding.task.service.{thriftscala => ots} import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.transformer.HasFlipInjectionParams import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.request._ import com.twitter.product_mixer.core.pipeline.HasPipelineCursor import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.configapi.Params case class FollowingQuery( override val params: Params, override val clientContext: ClientContext, override val pipelineCursor: Option[UrtOrderedCursor], override val requestedMaxResults: Option[Int], override val debugOptions: Option[DebugOptions], override val features: Option[FeatureMap], override val deviceContext: Option[DeviceContext], override val seenTweetIds: Option[Seq[Long]], override val dspClientContext: Option[dsp.DspClientContext]) extends PipelineQuery with HasPipelineCursor[UrtOrderedCursor] with HasDeviceContext with HasSeenTweetIds with HasFlipInjectionParams with HomeAdsQuery { override val product: Product = FollowingProduct override def withFeatureMap(features: FeatureMap): FollowingQuery = copy(features = Some(features)) override val timelineRequestParams: Option[TimelineRequestParams] = Some(TimelineRequestParams(homeTimelineType = Some(HomeTimelineType.HomeLatest))) // Fields below are used for FLIP Injection in Onboarding Task Service (OTS) override val displayLocation: ots.DisplayLocation = ots.DisplayLocation.HomeLatestTimeline override val rankingDisablerWithLatestControlsAvailable: Option[Boolean] = None override val isEmptyState: Option[Boolean] = None override val isFirstRequestAfterSignup: Option[Boolean] = None override val isEndOfTimeline: Option[Boolean] = None override val timelineId: Option[Long] = None } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.scala ================================================ package com.twitter.home_mixer.product.following.model import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.ExternalStringRegistry import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton class HomeMixerExternalStrings @Inject() ( @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) { val seeNewTweetsString = externalStringRegistryProvider.get().createProdString("SeeNewTweets") val tweetedString = externalStringRegistryProvider.get().createProdString("Tweeted") val muteUserString = externalStringRegistryProvider.get().createProdString("Feedback.muteUser") val muteUserConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.muteUserConfirmation") val blockUserString = externalStringRegistryProvider.get().createProdString("Feedback.blockUser") val blockUserConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.blockUserConfirmation") val unfollowUserString = externalStringRegistryProvider.get().createProdString("Feedback.unfollowUser") val unfollowUserConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.unfollowUserConfirmation") val reportTweetString = externalStringRegistryProvider.get().createProdString("Feedback.reportTweet") val genericString = externalStringRegistryProvider.get().createProdString("Feedback.generic") val genericConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.genericConfirmation") val relevantString = externalStringRegistryProvider.get().createProdString("Feedback.relevant") val relevantConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.relevantConfirmation") val dontLikeString = externalStringRegistryProvider.get().createProdString("Feedback.dontLike") val dontLikeConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.dontLikeConfirmation") val showFewerTweetsString = externalStringRegistryProvider.get().createProdString("Feedback.showFewerTweets") val showFewerTweetsConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.showFewerTweetsConfirmation") val showFewerRetweetsString = externalStringRegistryProvider.get().createProdString("Feedback.showFewerRetweets") val showFewerRetweetsConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.showFewerRetweetsConfirmation") val notRelevantString = externalStringRegistryProvider.get().createProdString("Feedback.notRelevant") val notRelevantConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.notRelevantConfirmation") val hatefulString = externalStringRegistryProvider.get().createProdString("Feedback.hateful") val hatefulConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.hatefulConfirmation") val boringString = externalStringRegistryProvider.get().createProdString("Feedback.boring") val boringConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.boringConfirmation") val confusingString = externalStringRegistryProvider.get().createProdString("Feedback.confusing") val confusingConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.confusingConfirmation") val clickbaitString = externalStringRegistryProvider.get().createProdString("Feedback.clickbait") val clickbaitConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.clickbaitConfirmation") val ragebaitString = externalStringRegistryProvider.get().createProdString("Feedback.ragebait") val ragebaitConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.ragebaitConfirmation") val regretString = externalStringRegistryProvider.get().createProdString("Feedback.regret") val regretConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.regretConfirmation") val neutralString = externalStringRegistryProvider.get().createProdString("Feedback.neutral") val neutralConfirmationString = externalStringRegistryProvider.get().createProdString("Feedback.neutralConfirmation") val seeMoreString = externalStringRegistryProvider.get().createProdString("PagedCarouselModule.showMoreText") val seeLessString = externalStringRegistryProvider.get().createProdString("PagedCarouselModule.showLessText") val socialContextOneUserLikedString = externalStringRegistryProvider.get().createProdString("SocialContext.oneUserLiked") val socialContextTwoUsersLikedString = externalStringRegistryProvider.get().createProdString("SocialContext.twoUsersLiked") val socialContextMoreUsersLikedString = externalStringRegistryProvider.get().createProdString("SocialContext.moreUsersLiked") val socialContextLikedByTimelineTitle = externalStringRegistryProvider.get().createProdString("SocialContext.likedByTimelineTitle") val socialContextOneUserFollowsString = externalStringRegistryProvider.get().createProdString("SocialContext.oneUserFollows") val socialContextTwoUsersFollowString = externalStringRegistryProvider.get().createProdString("SocialContext.twoUsersFollow") val socialContextMoreUsersFollowString = externalStringRegistryProvider.get().createProdString("SocialContext.moreUsersFollow") val socialContextFollowedByTimelineTitle = externalStringRegistryProvider.get().createProdString("SocialContext.followedByTimelineTitle") val socialContextYouMightLikeString = externalStringRegistryProvider.get().createProdString("SocialContext.youMightLike") val socialContextExtendedReply = externalStringRegistryProvider.get().createProdString("SocialContext.extendedReply") val socialContextReceivedReply = externalStringRegistryProvider.get().createProdString("SocialContext.receivedReply") val socialContextPopularInYourAreaString = externalStringRegistryProvider.get().createProdString("PopgeoTweet.socialProof") val ownedSubscribedListsModuleHeaderString = externalStringRegistryProvider.get().createProdString("OwnedSubscribedListModule.header") val CommunityToJoinHeaderString = externalStringRegistryProvider.get().createProdString("CommunityToJoinModule.header") val CommunityToJoinFooterString = externalStringRegistryProvider.get().createProdString("CommunityToJoinModule.footer") val RecommendedJobHeaderString = externalStringRegistryProvider.get().createProdString("RecommendedJobModule.header") val RecommendedJobFooterString = externalStringRegistryProvider.get().createProdString("RecommendedJobModule.footer") val RecommendedRecruitingOrganizationHeaderString = externalStringRegistryProvider .get().createProdString("RecommendedRecruitingOrganizationModule.header") val RecommendedRecruitingOrganizationFooterString = externalStringRegistryProvider .get().createProdString("RecommendedRecruitingOrganizationModule.footer") val BookmarksHeaderString = externalStringRegistryProvider.get().createProdString("RecentBookmarks.header") val PinnedTweetsHeaderString = externalStringRegistryProvider.get().createProdString("PinnedTweetsModule.header") val BroadcastedPinnedTweetSocialContextString = externalStringRegistryProvider.get().createProdString("BroadcastedPinnedTweet.context") val VideoCarouselHeaderString = externalStringRegistryProvider.get().createProdString("VideoCarouselModule.header") val VideoCarouselFooterString = externalStringRegistryProvider.get().createProdString("VideoCarouselModule.footer") val TrendingString = externalStringRegistryProvider.get().createProdString("Trending") val KeywordTrendsTweetCountDescriptionString = externalStringRegistryProvider.get().createProdString("KeywordTrends.tweetCountDescription") val NewsHeaderString = externalStringRegistryProvider.get().createProdString("News.header") val NewsFooterString = externalStringRegistryProvider.get().createProdString("News.footer") } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParam.scala ================================================ package com.twitter.home_mixer.product.following.param import com.twitter.conversions.DurationOps._ import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.WhoToFollowModuleDisplayType import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowUserDisplayType import com.twitter.product_mixer.core.functional_component.configapi.StaticParam import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.util.Duration object FollowingParam { val SupportedClientFSName = "following_supported_client" val StaticParamValueZero = StaticParam(0) val StaticParamValueFive = StaticParam(5) object ServerMaxResultsParam extends FSBoundedParam[Int]( name = "following_server_max_results", default = 100, min = 1, max = 500 ) object EnableWhoToFollowCandidatePipelineParam extends FSParam[Boolean]( name = "following_enable_who_to_follow", default = true ) object EnableFlipInjectionModuleCandidatePipelineParam extends FSParam[Boolean]( name = "following_enable_flip_inline_injection_module", default = true ) object EnablePostContextFeatureHydratorParam extends FSParam[Boolean]( name = "following_enable_post_context_feature_hydrator", default = false ) object WhoToFollowMinInjectionIntervalParam extends FSBoundedParam[Duration]( "following_who_to_follow_min_injection_interval_in_minutes", default = 1800.minutes, min = 0.minutes, max = 6000.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object WhoToFollowDisplayTypeIdParam extends FSEnumParam[WhoToFollowModuleDisplayType.type]( name = "following_enable_who_to_follow_display_type_id", default = WhoToFollowModuleDisplayType.Vertical, enum = WhoToFollowModuleDisplayType ) object WhoToFollowUserDisplayTypeIdParam extends FSEnumParam[WhoToFollowUserDisplayType.type]( name = "following_enable_who_to_follow_user_display_type_id", default = WhoToFollowUserDisplayType.User, enum = WhoToFollowUserDisplayType ) object WhoToFollowDisplayLocationParam extends FSParam[String]( name = "following_who_to_follow_display_location", default = "timeline_reverse_chron" ) object EnableFastAds extends FSParam[Boolean]( name = "following_enable_fast_ads", default = true ) object EnableDependentAdsParam extends FSParam[Boolean]( name = "following_enable_dependent_ads", default = true ) object EnableNavigationInstructionParam extends FSParam[Boolean]( name = "following_enable_navigation_instruction", default = false ) object ClearCache { object PtrEnableParam extends FSParam[Boolean]( name = "following_clear_cache_ptr_enable", default = false ) object ColdStartEnableParam extends FSParam[Boolean]( name = "following_clear_cache_cold_start_enable", default = false ) object WarmStartEnableParam extends FSParam[Boolean]( name = "following_clear_cache_warm_start_enable", default = false ) object ManualRefreshEnableParam extends FSParam[Boolean]( name = "following_clear_cache_manual_refresh_enable", default = false ) object NavigateEnableParam extends FSParam[Boolean]( name = "following_clear_cache_navigate_enable", default = false ) case object MinEntriesParam extends FSBoundedParam[Int]( name = "following_clear_cache_min_entries", default = 10, min = 0, max = 35 ) } object Navigation { object PtrEnableParam extends FSParam[Boolean]( name = "following_navigation_ptr_enable", default = false ) object ColdStartEnableParam extends FSParam[Boolean]( name = "following_navigation_cold_start_enable", default = false ) object WarmStartEnableParam extends FSParam[Boolean]( name = "following_navigation_warm_start_enable", default = false ) object ManualRefreshEnableParam extends FSParam[Boolean]( name = "following_navigation_manual_refresh_enable", default = false ) object NavigateEnableParam extends FSParam[Boolean]( name = "following_navigation_navigate_enable", default = false ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParamConfig.scala ================================================ package com.twitter.home_mixer.product.following.param import com.twitter.home_mixer.param.decider.DeciderKey import com.twitter.home_mixer.product.following.param.FollowingParam._ import com.twitter.product_mixer.core.product.ProductParamConfig import com.twitter.servo.decider.DeciderKeyName import javax.inject.Inject import javax.inject.Singleton @Singleton class FollowingParamConfig @Inject() () extends ProductParamConfig { override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableFollowingProduct override val supportedClientFSName: String = SupportedClientFSName override val booleanFSOverrides = Seq( ClearCache.PtrEnableParam, ClearCache.ColdStartEnableParam, ClearCache.WarmStartEnableParam, ClearCache.ManualRefreshEnableParam, ClearCache.NavigateEnableParam, EnableFlipInjectionModuleCandidatePipelineParam, EnableWhoToFollowCandidatePipelineParam, EnablePostContextFeatureHydratorParam, EnableFastAds, EnableDependentAdsParam, EnableNavigationInstructionParam, Navigation.PtrEnableParam, Navigation.ColdStartEnableParam, Navigation.WarmStartEnableParam, Navigation.ManualRefreshEnableParam, Navigation.NavigateEnableParam, ) override val boundedIntFSOverrides = Seq( ClearCache.MinEntriesParam, ServerMaxResultsParam ) override val stringFSOverrides = Seq(WhoToFollowDisplayLocationParam) override val boundedDurationFSOverrides = Seq(WhoToFollowMinInjectionIntervalParam) override val enumFSOverrides = Seq( WhoToFollowDisplayTypeIdParam, WhoToFollowUserDisplayTypeIdParam ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "ads-injection/lib/src/main/scala/com/twitter/goldfinch/api", "communities-mixer/thrift/src/main/thrift:thrift-scala", "events-recos/events-recos-service/src/main/thrift:events-recos-thrift-scala", "geoduck/util/country", "home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/entry_point_pivot", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/location", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/frame", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/pivot", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/trends_events", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/communities_to_join", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline/selector", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/jetfuel_entry_point", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/job", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/recruiting_organization", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_subscribe_module", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/cursor/timelines", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", "recruiting/candidate-service/src/main/thrift:thrift-scala", "recruiting/src/main/thrift:thrift-scala", "src/thrift/com/twitter/frigate/bookmarks:bookmarks-thrift-scala", "src/thrift/com/twitter/search:earlybird-scala", "strato/config/columns/events/entryPoint:entryPoint-strato-client", "strato/config/columns/events/experiences/grok:grok-strato-client", "strato/config/columns/trendsai/ordered:ordered-strato-client", "strato/config/src/thrift/com/twitter/strato/columns/jetfuel:jetfuel-scala", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", ], exports = [ "src/thrift/com/twitter/timelines/render:thrift-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.adserver.{thriftscala => ads} import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder import com.twitter.home_mixer.functional_component.feature_hydrator.NoAdsTierFeature import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate import com.twitter.home_mixer.functional_component.gate.ExcludeSyntheticUserGate import com.twitter.home_mixer.param.HomeGlobalParams import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.AdsNumOrganicItemsParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad.AdsCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator import com.twitter.product_mixer.component_library.gate.FeatureGate import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsCandidatePipelineConfigBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.ads.StaticAdsDisplayLocationBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter import com.twitter.product_mixer.core.gate.ParamNotGate import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouAdsCandidatePipelineBuilder @Inject() ( adsCandidatePipelineConfigBuilder: AdsCandidatePipelineConfigBuilder, adsCandidateSource: AdsProdThriftCandidateSource, advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ ForYouQuery, AdsCandidate ]) { private val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouAds") private val servedType = hmt.ServedType.ForYouPromoted private val clientEventInfoBuilder = ClientEventInfoBuilder( component = servedType.originalName, detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(servedType.name))) ) private val contextualTweetRefBuilder = ContextualTweetRefBuilder( TweetHydrationContext( safetyLevelOverride = Some(TimelineHomePromotedHydrationSafetyLevel), outerTweetContext = None )) private val decorator = UrtItemCandidateDecorator( AdsCandidateUrtItemBuilder( tweetClientEventInfoBuilder = Some(clientEventInfoBuilder), contextualTweetRefBuilder = Some(contextualTweetRefBuilder) )) private val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() ) def build(): AdsCandidatePipelineConfig[ForYouQuery] = adsCandidatePipelineConfigBuilder.build[ForYouQuery]( adsCandidateSource = adsCandidateSource, identifier = identifier, adsDisplayLocationBuilder = StaticAdsDisplayLocationBuilder(ads.DisplayLocation.TimelineHome), estimateNumOrganicItems = _.params(AdsNumOrganicItemsParam).toShort, gates = Seq( ParamNotGate( name = "AdsDisableInjectionBasedOnUserRole", param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam ), ExcludeSoftUserGate, ExcludeSyntheticUserGate, FeatureGate.fromNegatedFeature(NoAdsTierFeature) ), filters = Seq(ValidAdImpressionIdFilter), postFilterFeatureHydration = Seq( ParamGatedCandidateFeatureHydrator( EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, advertiserBrandSafetySettingsFeatureHydrator ) ), decorator = Some(decorator), alerts = alerts, urtRequest = Some(true) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.adserver.{thriftscala => ads} import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder import com.twitter.home_mixer.functional_component.feature_hydrator.NoAdsTierFeature import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate import com.twitter.home_mixer.functional_component.gate.ExcludeSyntheticUserGate import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature import com.twitter.home_mixer.param.HomeGlobalParams import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad.AdsCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator import com.twitter.product_mixer.component_library.gate.FeatureGate import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfigBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.ads.CountTruncatedItemCandidatesFromPipelines import com.twitter.product_mixer.component_library.pipeline.candidate.ads.StaticAdsDisplayLocationBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.ads.TruncatedPipelineScopedOrganicItems import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.gate.ParamNotGate import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouAdsDependentCandidatePipelineBuilder @Inject() ( adsCandidatePipelineConfigBuilder: AdsDependentCandidatePipelineConfigBuilder, adsCandidateSource: AdsProdThriftCandidateSource, advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ ForYouQuery, AdsCandidate ]) { private val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouAdsDependent") private val servedType = hmt.ServedType.ForYouPromoted private val MaxOrganicTweets = 35 private val clientEventInfoBuilder = ClientEventInfoBuilder( component = servedType.originalName, detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(servedType.name))) ) private val contextualTweetRefBuilder = ContextualTweetRefBuilder( TweetHydrationContext( safetyLevelOverride = Some(TimelineHomePromotedHydrationSafetyLevel), outerTweetContext = None )) private val decorator = UrtItemCandidateDecorator( AdsCandidateUrtItemBuilder( tweetClientEventInfoBuilder = Some(clientEventInfoBuilder), contextualTweetRefBuilder = Some(contextualTweetRefBuilder) )) private val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() ) def build( organicCandidatePipelines: CandidateScope ): AdsDependentCandidatePipelineConfig[ForYouQuery] = adsCandidatePipelineConfigBuilder.build[ForYouQuery]( adsCandidateSource = adsCandidateSource, identifier = identifier, adsDisplayLocationBuilder = StaticAdsDisplayLocationBuilder(ads.DisplayLocation.TimelineHome), getOrganicItems = TruncatedPipelineScopedOrganicItems( pipelines = organicCandidatePipelines, textFeature = TweetTextFeature, languageFeature = TweetLanguageFeature, ordering = CandidatesUtil.scoreOrdering, maxCount = MaxOrganicTweets ), countNumOrganicItems = CountTruncatedItemCandidatesFromPipelines(organicCandidatePipelines, MaxOrganicTweets), gates = Seq( ParamNotGate( name = "AdsDisableInjectionBasedOnUserRole", param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam ), ExcludeSoftUserGate, ExcludeSyntheticUserGate, FeatureGate.fromNegatedFeature(NoAdsTierFeature), NonEmptyCandidatesGate(organicCandidatePipelines) ), filters = Seq(ValidAdImpressionIdFilter), postFilterFeatureHydration = Seq( ParamGatedCandidateFeatureHydrator( EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, advertiserBrandSafetySettingsFeatureHydrator ) ), decorator = Some(decorator), alerts = alerts, urtRequest = Some(true) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouBookmarksCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.frigate.bookmarks.{thriftscala => t} import com.twitter.home_mixer.functional_component.decorator.urt.builder.TweetCarouselModuleCandidateDecorator import com.twitter.home_mixer.functional_component.decorator.urt.builder.TweetCarouselType import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.filter.TweetHydrationFilter import com.twitter.home_mixer.functional_component.filter.WeeklyBookmarkFilter import com.twitter.home_mixer.functional_component.gate.BookmarksTimeGate import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.product.for_you.candidate_source.BookmarksCandidateSource import com.twitter.home_mixer.product.for_you.param.ForYouParam.BookmarksModuleMinInjectionIntervalParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableBookmarksCandidatePipelineParam import com.twitter.home_mixer.product.for_you.response_transformer.BookmarksResponseFeatureTransformer import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelines.configapi.FSParam import com.twitter.timelineservice.model.rich.EntityIdType import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouBookmarksCandidatePipelineConfig @Inject() ( bookmarksTimeGate: BookmarksTimeGate, bookmarksCandidateSource: BookmarksCandidateSource, tweetypieFeatureHydrator: TweetypieFeatureHydrator, tweetCarouselModuleCandidateDecorator: TweetCarouselModuleCandidateDecorator) extends CandidatePipelineConfig[ PipelineQuery, Long, t.BookmarkedTweet, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouBookmarks") override val supportedClientParam: Option[FSParam[Boolean]] = Some(EnableBookmarksCandidatePipelineParam) override val gates: Seq[Gate[PipelineQuery]] = Seq( DefinedUserIdGate, RateLimitGate, bookmarksTimeGate, TimelinesPersistenceStoreLastInjectionGate( BookmarksModuleMinInjectionIntervalParam, EntityIdType.BookmarksModule ) ) override val queryTransformer: CandidatePipelineQueryTransformer[PipelineQuery, Long] = query => query.getRequiredUserId override val candidateSource: BaseCandidateSource[Long, t.BookmarkedTweet] = bookmarksCandidateSource override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[t.BookmarkedTweet] ] = Seq(BookmarksResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ t.BookmarkedTweet, TweetCandidate ] = tweet => TweetCandidate(tweet.tweetId) override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[PipelineQuery, TweetCandidate, _] ] = Seq(tweetypieFeatureHydrator) override val filters: Seq[Filter[PipelineQuery, TweetCandidate]] = Seq(TweetHydrationFilter, WeeklyBookmarkFilter) override val decorator: Option[ CandidateDecorator[PipelineQuery, TweetCandidate] ] = Some(tweetCarouselModuleCandidateDecorator.build(TweetCarouselType.BookmarkedTweets)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouCommunitiesToJoinCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.communities_mixer.{thriftscala => cmt} import com.twitter.home_mixer.functional_component.decorator.urt.builder.FeedbackStrings import com.twitter.home_mixer.functional_component.filter.DropMaxCandidatesFilter import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.for_you.param.ForYouParam.CommunitiesToJoinDisplayTypeIdParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.CommunitiesToJoinMinInjectionIntervalParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableCommunitiesToJoinCandidatePipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.MaxCommunitiesToJoinCandidatesParam import com.twitter.product_mixer.component_library.candidate_source.communities.CommunitiesToJoinCandidateSource import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.IsAvailableToJoinFeature import com.twitter.product_mixer.component_library.feature_hydrator.candidate.communities.IsAvailableToJoinFeatureHydrator import com.twitter.product_mixer.component_library.filter.FeatureFilter import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate import com.twitter.product_mixer.component_library.gate.communities.CommunitiesJoinLimitGate import com.twitter.product_mixer.component_library.model.candidate.CommunityCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.communities_to_join.CommunitiesToJoinCandidateDecorator import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextMarshaller import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.HasPipelineCursor import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.configapi.FSParam import com.twitter.timelineservice.model.rich.EntityIdType import com.twitter.timelineservice.suggests.{thriftscala => st} import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton class ForYouCommunitiesToJoinCandidatePipelineConfig @Inject() ( communitiesToJoinCandidateSource: CommunitiesToJoinCandidateSource, communitiesJoinLimitGate: CommunitiesJoinLimitGate, isAvailableToJoinFeatureHydrator: IsAvailableToJoinFeatureHydrator, @ProductScoped stringCenterProvider: Provider[StringCenter], externalStrings: HomeMixerExternalStrings, feedbackStrings: FeedbackStrings) extends CandidatePipelineConfig[ PipelineQuery with HasPipelineCursor[_], cmt.CommunitiesMixerRequest, Long, CommunityCandidate ] { private val stringCenter = stringCenterProvider.get() private val MaxCommunities = 10 override val supportedClientParam: Option[FSParam[Boolean]] = Some( EnableCommunitiesToJoinCandidatePipelineParam) override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouCommunitiesToJoin") private val IsAvailableToJoinFilterId = "IsAvailableToJoin" override val gates = Seq( DefinedUserIdGate, RateLimitGate, TimelinesPersistenceStoreLastInjectionGate( CommunitiesToJoinMinInjectionIntervalParam, EntityIdType.CommunityModule ), DismissFatigueGate(st.SuggestType.CommunityToJoin, DismissInfoFeature), communitiesJoinLimitGate ) override val queryTransformer: CandidatePipelineQueryTransformer[ PipelineQuery, cmt.CommunitiesMixerRequest ] = { query => cmt.CommunitiesMixerRequest( clientContext = ClientContextMarshaller(query.clientContext), product = cmt.Product.CommunityRecs, productContext = Some( cmt.ProductContext.CommunityRecs( cmt.CommunityRecs(displayFormat = cmt.DisplayFormat.Module))), cursor = None, maxResults = Some(MaxCommunities), debugParams = None, displayLocation = None ) } override val candidateSource: CandidateSource[cmt.CommunitiesMixerRequest, Long] = communitiesToJoinCandidateSource override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[PipelineQuery, CommunityCandidate, _] ] = Seq(isAvailableToJoinFeatureHydrator) override def filters: Seq[Filter[PipelineQuery, CommunityCandidate]] = Seq( FeatureFilter .fromFeature(FilterIdentifier(IsAvailableToJoinFilterId), IsAvailableToJoinFeature), DropMaxCandidatesFilter(MaxCommunitiesToJoinCandidatesParam) ) override val resultTransformer: CandidatePipelineResultsTransformer[Long, CommunityCandidate] = { communityResult => CommunityCandidate(id = communityResult) } override val decorator: Option[ CandidateDecorator[PipelineQuery, CommunityCandidate] ] = { Some( CommunitiesToJoinCandidateDecorator( moduleDisplayTypeParam = CommunitiesToJoinDisplayTypeIdParam, stringCenter = stringCenter, headerString = externalStrings.CommunityToJoinHeaderString, footerString = Some(externalStrings.CommunityToJoinFooterString), seeLessOftenString = Some(feedbackStrings.seeLessOftenFeedbackString), seeLessOftenConfirmationString = Some(feedbackStrings.seeLessOftenConfirmationFeedbackString) )) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouConversationServiceCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.candidate_pipeline.ConversationServiceResponseFeatureTransformer import com.twitter.home_mixer.functional_component.decorator.HomeConversationServiceCandidateDecorator import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter import com.twitter.home_mixer.functional_component.filter.PreviouslyServedTweetsFilter import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata import com.twitter.product_mixer.component_library.filter.FeatureFilter import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter import com.twitter.product_mixer.component_library.gate.NoCandidatesGate import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.BaseGate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.functional_component.transformer.DependentCandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig import javax.inject.Inject import javax.inject.Singleton /** * Candidate Pipeline Config that fetches Tweet ancestors from Conversation Service Candidate Source */ @Singleton class ForYouConversationServiceCandidatePipelineConfig @Inject() ( forYouScoredTweetsCandidatePipelineConfig: ForYouScoredTweetsCandidatePipelineConfig, forYouTimelineScorerCandidatePipelineConfig: ForYouTimelineScorerCandidatePipelineConfig, conversationServiceCandidateSource: ConversationServiceCandidateSource, tweetypieFeatureHydrator: TweetypieFeatureHydrator, namesFeatureHydrator: NamesFeatureHydrator, invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) extends DependentCandidatePipelineConfig[ ForYouQuery, ConversationServiceCandidateSourceRequest, TweetWithConversationMetadata, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouConversationService") override val gates: Seq[BaseGate[ForYouQuery]] = Seq( NoCandidatesGate( SpecificPipelines( forYouTimelineScorerCandidatePipelineConfig.identifier, forYouScoredTweetsCandidatePipelineConfig.identifier ) ), NonEmptySeqFeatureGate(TimelineServiceTweetsFeature) ) override val candidateSource: BaseCandidateSource[ ConversationServiceCandidateSourceRequest, TweetWithConversationMetadata ] = conversationServiceCandidateSource override val queryTransformer: DependentCandidatePipelineQueryTransformer[ ForYouQuery, ConversationServiceCandidateSourceRequest ] = { (query, candidates) => val timelineServiceTweets = query.features .map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty)).getOrElse(Seq.empty) val tweetsWithConversationMetadata = timelineServiceTweets.map { id => TweetWithConversationMetadata( tweetId = id, userId = None, sourceTweetId = None, sourceUserId = None, inReplyToTweetId = None, conversationId = None, ancestors = Seq.empty ) } ConversationServiceCandidateSourceRequest(tweetsWithConversationMetadata) } override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[TweetWithConversationMetadata] ] = Seq(ConversationServiceResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ TweetWithConversationMetadata, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] ] = Seq( InNetworkFeatureHydrator, tweetypieFeatureHydrator ) override def filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( PreviouslyServedTweetsFilter, RetweetDeduplicationFilter, FeatureFilter.fromFeature(FilterIdentifier("TweetypieHydrated"), IsHydratedFeature), PredicateFeatureFilter.fromPredicate( FilterIdentifier("QuotedTweetDropped"), shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } ), invalidSubscriptionTweetFilter, InvalidConversationModuleFilter ) override val postFilterFeatureHydration: Seq[ BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] ] = Seq(namesFeatureHydrator) override val decorator: Option[CandidateDecorator[ForYouQuery, TweetCandidate]] = HomeConversationServiceCandidateDecorator(homeFeedbackActionInfoBuilder) override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouEntryPointPivotCandidatePipelineBuilder.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.decorator.EntryPointPivotModuleCandidateDecorator import com.twitter.home_mixer.functional_component.decorator.StrEntryPointPivotCategoryText import com.twitter.home_mixer.product.for_you.gate.FollowingSportsUserGate import com.twitter.product_mixer.component_library.candidate_source.entry_point_pivot.EntryPointPivotCandidateSource import com.twitter.product_mixer.component_library.candidate_source.entry_point_pivot.GrokEntryPointPivotCandidateSource import com.twitter.product_mixer.component_library.model.candidate.pivot.PivotCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.Param import com.twitter.timelines.render.{thriftscala => t} import com.twitter.util.Duration import javax.inject.Inject import javax.inject.Singleton object ForYouEntryPointPivotCandidatePipelineBuilder { sealed abstract class EntryPointPivotType() object EntryPointPivotType { case object Events extends EntryPointPivotType() case object Grok extends EntryPointPivotType() } } @Singleton class ForYouEntryPointPivotCandidatePipelineBuilder @Inject() ( entryPointPivotCandidateSource: EntryPointPivotCandidateSource, grokEntryPointPivotCandidateSource: GrokEntryPointPivotCandidateSource) { import ForYouEntryPointPivotCandidatePipelineBuilder._ def build( entryPointPivotType: EntryPointPivotType, supportedClientParam: FSParam[Boolean], pivotMinInjectionIntervalParam: Param[Duration] ): ForYouEntryPointPivotCandidatePipelineConfig = new ForYouEntryPointPivotCandidatePipelineConfig( identifier = getCandidateIdentifier(entryPointPivotType), supportedClientParam = Some(supportedClientParam), candidateSource = getCandidateSource(entryPointPivotType), decorator = Some(getDecorator(entryPointPivotType)), pivotMinInjectionIntervalParam = pivotMinInjectionIntervalParam, extraGates = getPipelineGates(entryPointPivotType) ) private def getCandidateSource(entryPointPivotType: EntryPointPivotType): CandidateSource[ StratoKeyView[String, Unit], t.Pivot ] = entryPointPivotType match { case EntryPointPivotType.Grok => grokEntryPointPivotCandidateSource case EntryPointPivotType.Events => entryPointPivotCandidateSource } private def getDecorator( entryPointPivotType: EntryPointPivotType ): CandidateDecorator[PipelineQuery, PivotCandidate] = entryPointPivotType match { case EntryPointPivotType.Grok => EntryPointPivotModuleCandidateDecorator( component = "grok_pivot", headerText = StrEntryPointPivotCategoryText("") ).moduleDecorator case EntryPointPivotType.Events => EntryPointPivotModuleCandidateDecorator( component = "sport_entry_point", headerText = StrEntryPointPivotCategoryText("Play now!") ).moduleDecorator } private def getPipelineGates(entryPointPivotType: EntryPointPivotType): Seq[Gate[PipelineQuery]] = entryPointPivotType match { case EntryPointPivotType.Grok => Seq.empty case EntryPointPivotType.Events => Seq(FollowingSportsUserGate) } private def getCandidateIdentifier( entryPointPivotType: EntryPointPivotType ): CandidatePipelineIdentifier = CandidatePipelineIdentifier(s"ForYouEntryPointPivot${entryPointPivotType.toString}") } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouEntryPointPivotCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate import com.twitter.product_mixer.component_library.model.candidate.pivot.PivotCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.HasPipelineCursor import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.Param import com.twitter.timelines.render.{thriftscala => t} import com.twitter.timelineservice.model.rich.EntityIdType import com.twitter.util.Duration import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouEntryPointPivotCandidatePipelineConfig @Inject() ( override val identifier: CandidatePipelineIdentifier, override val supportedClientParam: Option[FSParam[Boolean]], override val candidateSource: CandidateSource[ StratoKeyView[String, Unit], t.Pivot ], override val decorator: Option[CandidateDecorator[PipelineQuery, PivotCandidate]], pivotMinInjectionIntervalParam: Param[Duration], extraGates: Seq[Gate[PipelineQuery]]) extends CandidatePipelineConfig[ PipelineQuery with HasPipelineCursor[_], StratoKeyView[String, Unit], t.Pivot, PivotCandidate ] { override val gates = Seq( DefinedUserIdGate, RateLimitGate, TimelinesPersistenceStoreLastInjectionGate( pivotMinInjectionIntervalParam, EntityIdType.EntryPointPivot ) ) ++ extraGates override val queryTransformer: CandidatePipelineQueryTransformer[ PipelineQuery, StratoKeyView[String, Unit] ] = { query => StratoKeyView( key = query.getCountryCode.getOrElse("US"), view = None ) } override val resultTransformer: CandidatePipelineResultsTransformer[ t.Pivot, PivotCandidate ] = { sourceResult => PivotCandidate( // We only expect one item per "messageprompt" entryNamespace per URT response. // As a result id=0L will always be unique in the entryNamespace. id = identifier.name, url = sourceResult.url, displayType = sourceResult.displayType, titleText = sourceResult.titleText, detailText = sourceResult.detailText, image = sourceResult.image, badge = sourceResult.badge, categoryText = sourceResult.categoryText, detailTextImage = sourceResult.detailTextImage, element = None ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouExplorationTweetsCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.filter.TweetHydrationFilter import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.home_mixer.model.HomeFeatures.TLSOriginalTweetsWithConfirmedAuthorFeature import com.twitter.home_mixer.model.HomeFeatures.TweetAuthorFollowersFeature import com.twitter.home_mixer.product.for_you.feature_hydrator.TweetAuthorFeatureHydrator import com.twitter.home_mixer.product.for_you.feature_hydrator.TweetEngagementsFeatureHydrator import com.twitter.home_mixer.product.for_you.feature_hydrator.TweetAuthorFollowersFeatureHydrator import com.twitter.home_mixer.product.for_you.feature_hydrator.TweetEngagementCountsFeature import com.twitter.home_mixer.product.for_you.feature_hydrator.TweetEngagementCounts import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableExplorationTweetsCandidatePipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.InNetworkExplorationTweetsMinInjectionIntervalParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.ExplorationTweetsMaxFollowerCountParam import com.twitter.home_mixer.product.for_you.response_transformer.ExplorationTweetResponseFeatureTransformer import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.candidate_source.PassthroughCandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelines.configapi.FSParam import com.twitter.home_mixer.functional_component.gate.RecentlyServedByServedTypeGate import com.twitter.home_mixer.product.for_you.gate.UserFollowingRangeGate import javax.inject.Inject import javax.inject.Singleton import scala.util.Random @Singleton class ForYouExplorationTweetsCandidatePipelineConfig @Inject() ( tweetAuthorFeatureHydrator: TweetAuthorFeatureHydrator, tweetEngagementsFeatureHydrator: TweetEngagementsFeatureHydrator, tweetAuthorFollowersFeatureHydrator: TweetAuthorFollowersFeatureHydrator, tweetypieFeatureHydrator: TweetypieFeatureHydrator, homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) extends CandidatePipelineConfig[ ForYouQuery, ForYouQuery, Long, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouExplorationTweets") override val supportedClientParam: Option[FSParam[Boolean]] = Some( EnableExplorationTweetsCandidatePipelineParam) override val queryTransformer: CandidatePipelineQueryTransformer[ ForYouQuery, ForYouQuery ] = identity override val queryFeatureHydration: Seq[QueryFeatureHydrator[PipelineQuery]] = Seq( tweetAuthorFeatureHydrator, tweetEngagementsFeatureHydrator ) override val queryFeatureHydrationPhase2: Seq[QueryFeatureHydrator[PipelineQuery]] = Seq( tweetAuthorFollowersFeatureHydrator ) override val gates: Seq[Gate[PipelineQuery]] = Seq( RecentlyServedByServedTypeGate( InNetworkExplorationTweetsMinInjectionIntervalParam, hmt.ServedType.ForYouExploration ), UserFollowingRangeGate ) override def candidateSource: CandidateSource[ForYouQuery, Long] = PassthroughCandidateSource( identifier = CandidateSourceIdentifier("ForYouExplorationTweets"), candidateExtractor = { query => val followedUserIds = query.features .map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)) .getOrElse(Seq.empty).toSet val servedAuthorIds = query.features .map(_.getOrElse(PersistenceEntriesFeature, Seq.empty)).getOrElse(Seq.empty) .flatMap(_.entries.flatMap(_.sourceAuthorIds)) .toSet val explorationAuthors = followedUserIds -- servedAuthorIds val engagementCounts = query.features .map(_.getOrElse(TweetEngagementCountsFeature, Map.empty[Long, TweetEngagementCounts])) .getOrElse(Map.empty) val maxFollowerCount = query.params(ExplorationTweetsMaxFollowerCountParam) val tweetAuthorFollowers = query.features .map(_.getOrElse(TweetAuthorFollowersFeature, Map.empty[Long, Option[Long]])) .getOrElse(Map.empty) val groupedByAuthor = query.features .map(_.getOrElse(TLSOriginalTweetsWithConfirmedAuthorFeature, Seq.empty)) .toSeq.flatten.filter { case (tweetId, authorId) => explorationAuthors.contains(authorId) && { val followerCount = tweetAuthorFollowers.get(tweetId).flatten.getOrElse(0L) followerCount <= maxFollowerCount } } .groupBy { case (_, authorId) => authorId } val bestPostsByAuthor = groupedByAuthor.flatMap { case (authorId, candidates) => val candidateWithEngagement = candidates.map { case (tweetId, _) => val engagement = engagementCounts.get(tweetId) val totalEngagement = engagement .map { counts => counts.favoriteCount.getOrElse(0L) + counts.replyCount.getOrElse(0L) + counts.retweetCount.getOrElse(0L) + counts.quoteCount.getOrElse(0L) + counts.bookmarkCount.getOrElse(0L) }.getOrElse(0L) (tweetId, totalEngagement) } if (candidateWithEngagement.nonEmpty) { val bestPost = candidateWithEngagement.maxBy(_._2) Some(bestPost._1) } else None } Random.shuffle(bestPostsByAuthor.toSeq).take(1) } ) override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[Long] ] = Seq(ExplorationTweetResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[Long, TweetCandidate] = { sourceResult => TweetCandidate(id = sourceResult) } override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] ] = Seq(tweetypieFeatureHydrator) override val filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq(TweetHydrationFilter) override val decorator: Option[CandidateDecorator[PipelineQuery, TweetCandidate]] = { val clientEventInfoBuilder = ClientEventInfoBuilder[PipelineQuery, TweetCandidate]( hmt.ServedType.ForYouExploration.originalName) val tweetItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( clientEventInfoBuilder = clientEventInfoBuilder, feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder), ) Some(UrtItemCandidateDecorator(tweetItemBuilder)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouJetfuelFrameFrameCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.strato.columns.jetfuel.thriftscala.JetfuelRouteData import com.twitter.home_mixer.product.for_you.candidate_source.JetfuelFrameCandidateSource import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableJetfuelFramePipelineParam import com.twitter.home_mixer.product.for_you.gate.FollowingSportsUserGate import com.twitter.product_mixer.component_library.pipeline.candidate.jetfuel_entry_point.JetfuelCandidateFeatureTransformer import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.frame.FrameCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.gate.FirstPageGate import com.twitter.product_mixer.component_library.model.candidate.FrameCandidate import com.twitter.product_mixer.component_library.feature_hydrator.candidate.frame.JetfuelPayloadFeatureHydrator import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.gate.ParamGate import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.HasPipelineCursor import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.strato.generated.client.events.entryPoint.JetfuelEntryPointByCountryClientColumn import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouJetfuelFrameCandidatePipelineConfig @Inject() ( jetfuelFrameCandidateSource: JetfuelFrameCandidateSource, jetfuelPayloadFeatureHydrator: JetfuelPayloadFeatureHydrator) extends CandidatePipelineConfig[ PipelineQuery with HasPipelineCursor[_], StratoKeyView[ JetfuelEntryPointByCountryClientColumn.Key, JetfuelEntryPointByCountryClientColumn.View ], JetfuelRouteData, FrameCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouJetfuelFrame") private val FrameId = "ForYouJetfuelFrame" override val candidateSource: JetfuelFrameCandidateSource = jetfuelFrameCandidateSource override val postFilterFeatureHydration: Seq[ BaseCandidateFeatureHydrator[PipelineQuery, FrameCandidate, _] ] = Seq(jetfuelPayloadFeatureHydrator) override val gates: Seq[Gate[PipelineQuery with HasPipelineCursor[_]]] = Seq( FirstPageGate, ParamGate(name = "JetfuelFrameEnabled", param = EnableJetfuelFramePipelineParam), FollowingSportsUserGate ) override val queryTransformer: CandidatePipelineQueryTransformer[ PipelineQuery, StratoKeyView[ JetfuelEntryPointByCountryClientColumn.Key, JetfuelEntryPointByCountryClientColumn.View ] ] = { query => StratoKeyView(key = query.getCountryCode.getOrElse("US"), view = None) } override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[JetfuelRouteData] ] = Seq(JetfuelCandidateFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ JetfuelRouteData, FrameCandidate ] = { sourceResult => FrameCandidate(id = sourceResult.route) } override val decorator: Option[ CandidateDecorator[PipelineQuery, FrameCandidate] ] = { Some( UrtItemCandidateDecorator( FrameCandidateUrtItemBuilder( frameId = FrameId, clientEventInfoBuilder = None ) ) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouKeywordTrendsCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.events.recos.thriftscala.GetUnfiedCandidatesRequest import com.twitter.events.recos.{thriftscala => t} import com.twitter.home_mixer.functional_component.decorator.KeywordTrendsModuleCandidateDecorator import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.product.for_you.candidate_source.TrendCandidate import com.twitter.home_mixer.product.for_you.candidate_source.UnifiedTrendsCandidateSource import com.twitter.home_mixer.product.for_you.filter.PromotedTrendFilter import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableKeywordTrendsParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.KeywordTrendsModuleMinInjectionIntervalParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.MaxNumberKeywordTrendsParam import com.twitter.home_mixer.product.for_you.query_transformer.UnifiedCandidatesQueryTransformer import com.twitter.home_mixer.product.for_you.response_transformer.KeywordTrendsFeatureTransformer import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedTrendCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelines.configapi.FSParam import com.twitter.timelineservice.model.rich.EntityIdType import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouKeywordTrendsCandidatePipelineConfig @Inject() ( unifiedTrendsCandidateSource: UnifiedTrendsCandidateSource, keywordTrendsModuleCandidateDecorator: KeywordTrendsModuleCandidateDecorator) extends CandidatePipelineConfig[ PipelineQuery, t.GetUnfiedCandidatesRequest, TrendCandidate, UnifiedTrendCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier( "ForYouKeywordTrends") override val supportedClientParam: Option[FSParam[Boolean]] = Some(EnableKeywordTrendsParam) override val gates = Seq( RateLimitGate, DefinedUserIdGate, TimelinesPersistenceStoreLastInjectionGate( KeywordTrendsModuleMinInjectionIntervalParam, EntityIdType.Trends ) ) override val queryTransformer: CandidatePipelineQueryTransformer[ PipelineQuery, GetUnfiedCandidatesRequest ] = UnifiedCandidatesQueryTransformer( maxResultsParam = MaxNumberKeywordTrendsParam, candidatePipelineIdentifier = identifier) override val candidateSource: CandidateSource[ t.GetUnfiedCandidatesRequest, TrendCandidate ] = unifiedTrendsCandidateSource override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[TrendCandidate] ] = Seq(KeywordTrendsFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ TrendCandidate, UnifiedTrendCandidate ] = { result => UnifiedTrendCandidate(id = result.candidate.trendName) } override val filters: Seq[Filter[PipelineQuery, UnifiedTrendCandidate]] = Seq(PromotedTrendFilter) override val decorator: Option[CandidateDecorator[PipelineQuery, UnifiedTrendCandidate]] = Some(keywordTrendsModuleCandidateDecorator.moduleDecorator) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouMixerPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.clientapp.{thriftscala => ca} import com.twitter.finagle.stats.StatsReceiver import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig import com.twitter.home_mixer.candidate_pipeline.VerifiedPromptCandidatePipelineConfig import com.twitter.home_mixer.functional_component.feature_hydrator._ import com.twitter.home_mixer.functional_component.selector.UpdateConversationModuleId import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration import com.twitter.home_mixer.functional_component.side_effect._ import com.twitter.home_mixer.param.HomeGlobalParams.EnableSSPAdsBrandSafetySettingsFeatureHydratorParam import com.twitter.home_mixer.param.HomeGlobalParams.EnableUserActionsShadowScribeParam import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.for_you.feature_hydrator.DisplayedGrokTopicQueryFeatureHydrator import com.twitter.home_mixer.product.for_you.feature_hydrator.FollowingSportsAccountsQueryFeatureHydrator import com.twitter.home_mixer.product.for_you.feature_hydrator.TimelineServiceTweetsQueryFeatureHydrator import com.twitter.home_mixer.product.for_you.feature_hydrator.ViewerHasJobRecommendationsFeatureHydrator import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableAdsDebugParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableEntryPointPivotParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableFlipInjectionModuleCandidatePipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableFollowedGrokTopicsHydrationParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableForYouAppUpsellParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableForYouTimelineAdsSurface import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableForYouTopicSelectorParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableGrokEntryPointPivotParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.EntryPointPivotMinInjectionIntervalParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.ExplorationTweetsTimelinePosition import com.twitter.home_mixer.product.for_you.param.ForYouParam.ForYouAppUpsellJetfuelRouteParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.ForYouAppUpsellPosition import com.twitter.home_mixer.product.for_you.param.ForYouParam.ForYouTopicSelectorJetfuelRouteParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.ForYouTopicSelectorPosition import com.twitter.home_mixer.product.for_you.param.ForYouParam.GrokEntryPointPivotMinInjectionIntervalParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.GrokPivotModuleTimelinePosition import com.twitter.home_mixer.product.for_you.param.ForYouParam.MaxNumberExplorationTweetsParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.RelevancePromptTweetPositionParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.ServerMaxResultsParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.StaticParamValueFive import com.twitter.home_mixer.product.for_you.param.ForYouParam.StaticParamValueZero import com.twitter.home_mixer.product.for_you.param.ForYouParam.SuperbowlModuleTimelinePosition import com.twitter.home_mixer.product.for_you.param.ForYouParam.TuneFeedTimelinePosition import com.twitter.home_mixer.product.for_you.param.ForYouParam.VideoCarouselNumCandidates import com.twitter.home_mixer.product.for_you.param.ForYouParam.VideoCarouselNumTweetCandidatesToDedupeAgainstParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.VideoCarouselTimelinePosition import com.twitter.home_mixer.product.for_you.selector.DebugUpdateSortAdsResult import com.twitter.home_mixer.product.for_you.selector.RemoveDuplicateCandidatesOutsideModule import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateFeatureKeysKafkaSideEffectBuilder import com.twitter.home_mixer.product.for_you.side_effect.ServedStatsSideEffect import com.twitter.home_mixer.product.for_you.side_effect.VideoServedStatsSideEffect import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.feature_hydrator.query.ads.SSPAdsBrandSafetySettingsFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.AsyncParamGatedQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptCandidatePipelineConfigBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.selector.FlipPromptDynamicInsertionPosition import com.twitter.product_mixer.component_library.pipeline.candidate.jetfuel_entry_point.JetfuelCandidatePipelineConfigBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfig import com.twitter.product_mixer.component_library.selector.DropDuplicateCandidates import com.twitter.product_mixer.component_library.selector.DropMaxCandidates import com.twitter.product_mixer.component_library.selector.DropMaxModuleItemCandidates import com.twitter.product_mixer.component_library.selector.DropModuleTooFewModuleItemResults import com.twitter.product_mixer.component_library.selector.DropOrthogonalCandidates import com.twitter.product_mixer.component_library.selector.IdAndClassDuplicationKey import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.component_library.selector.InsertDynamicPositionResults import com.twitter.product_mixer.component_library.selector.InsertFixedPositionResults import com.twitter.product_mixer.component_library.selector.PickFirstCandidateMerger import com.twitter.product_mixer.component_library.selector.SelectConditionally import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates import com.twitter.product_mixer.component_library.selector.UpdateSortModuleItemCandidates import com.twitter.product_mixer.component_library.selector.ads.AdsInjector import com.twitter.product_mixer.component_library.selector.ads.InsertAdResults import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines import com.twitter.product_mixer.core.functional_component.configapi.StaticParam import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.gate.ParamGate import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.pipeline.FailOpenPolicy import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.render.{thriftscala => urt} import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton class ForYouMixerPipelineConfig @Inject() ( forYouAdsCandidatePipelineBuilder: ForYouAdsCandidatePipelineBuilder, forYouCommunitiesToJoinCandidatePipelineConfig: ForYouCommunitiesToJoinCandidatePipelineConfig, forYouScoredTweetsCandidatePipelineConfig: ForYouScoredTweetsCandidatePipelineConfig, forYouWhoToFollowCandidatePipelineConfigBuilder: ForYouWhoToFollowCandidatePipelineConfigBuilder, forYouWhoToSubscribeCandidatePipelineConfigBuilder: ForYouWhoToSubscribeCandidatePipelineConfigBuilder, forYouEntryPointPivotCandidatePipelineBuilder: ForYouEntryPointPivotCandidatePipelineBuilder, forYouRecommendedJobsCandidatePipelineConfig: ForYouRecommendedJobsCandidatePipelineConfig, forYouRecommendedRecruitingOrganizationsCandidatePipelineConfig: ForYouRecommendedRecruitingOrganizationsCandidatePipelineConfig, forYouBookmarksCandidatePipelineConfig: ForYouBookmarksCandidatePipelineConfig, forYouExplorationTweetsCandidatePipelineConfig: ForYouExplorationTweetsCandidatePipelineConfig, forYouJetfuelFrameCandidatePipelineConfig: ForYouJetfuelFrameCandidatePipelineConfig, forYouPinnedTweetsCandidatePipelineConfig: ForYouPinnedTweetsCandidatePipelineConfig, forYouStoriesCandidatePipelineConfig: ForYouStoriesCandidatePipelineConfig, flipPromptCandidatePipelineConfigBuilder: FlipPromptCandidatePipelineConfigBuilder, forYouKeywordTrendsCandidatePipelineConfig: ForYouKeywordTrendsCandidatePipelineConfig, forYouScoredVideoTweetsCandidatePipelineConfig: ForYouScoredVideoTweetsCandidatePipelineConfig, forYouRelevancePromptCandidatePipelineConfig: ForYouRelevancePromptCandidatePipelineConfig, editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, forYouTuneFeedCandidatePipelineConfig: ForYouTuneFeedCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[ForYouQuery], forYouTweetPreviewsCandidatePipelineConfig: ForYouTweetPreviewsCandidatePipelineConfig, verifiedPromptCandidatePipelineConfig: VerifiedPromptCandidatePipelineConfig, jetfuelCandidatePipelineConfigBuilder: JetfuelCandidatePipelineConfigBuilder, dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, followingSportsAccountsQueryFeatureHydrator: FollowingSportsAccountsQueryFeatureHydrator, gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, impressionBloomFilterQueryFeatureHydrator: ImpressionBloomFilterQueryFeatureHydrator, persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, rateLimitQueryFeatureHydrator: RateLimitQueryFeatureHydrator, requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ForYouQuery], timelineServiceTweetsQueryFeatureHydrator: TimelineServiceTweetsQueryFeatureHydrator, previewCreatorsQueryFeatureHydrator: PreviewCreatorsQueryFeatureHydrator, sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, userSubscriptionQueryFeatureHydrator: UserSubscriptionQueryFeatureHydrator, displayedGrokTopicQueryFeatureHydrator: DisplayedGrokTopicQueryFeatureHydrator, sspAdsBrandSafetySettingsFeatureHydrator: SSPAdsBrandSafetySettingsFeatureHydrator, viewerHasJobRecommendationsFeatureHydrator: ViewerHasJobRecommendationsFeatureHydrator, userActionsArrayByteQueryFeatureHydrator: UserActionsArrayByteQueryFeatureHydrator, adsInjector: AdsInjector, updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, homeScribeServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, servedCandidateFeatureKeysKafkaSideEffectBuilder: ServedCandidateFeatureKeysKafkaSideEffectBuilder, clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter], urtTransportMarshaller: UrtTransportMarshaller, @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean, statsReceiver: StatsReceiver) extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse] { override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ForYou") private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ForYouQuery]] = Seq( rateLimitQueryFeatureHydrator, requestQueryFeatureHydrator, persistenceStoreQueryFeatureHydrator, impressionBloomFilterQueryFeatureHydrator, timelineServiceTweetsQueryFeatureHydrator, previewCreatorsQueryFeatureHydrator, sgsFollowedUsersQueryFeatureHydrator, gizmoduckUserQueryFeatureHydrator, viewerHasJobRecommendationsFeatureHydrator, userSubscriptionQueryFeatureHydrator, ParamGatedQueryFeatureHydrator( EnableFollowedGrokTopicsHydrationParam, displayedGrokTopicQueryFeatureHydrator ), ParamGatedQueryFeatureHydrator( EnableSSPAdsBrandSafetySettingsFeatureHydratorParam, sspAdsBrandSafetySettingsFeatureHydrator ), ParamGatedQueryFeatureHydrator( EnableEntryPointPivotParam, followingSportsAccountsQueryFeatureHydrator ), AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), AsyncParamGatedQueryFeatureHydrator( EnableUserActionsShadowScribeParam, MixerPipelineConfig.resultSelectorsStep, userActionsArrayByteQueryFeatureHydrator ) ) private val forYouAdsCandidatePipelineConfig = forYouAdsCandidatePipelineBuilder.build() private val forYouWhoToFollowCandidatePipelineConfig = forYouWhoToFollowCandidatePipelineConfigBuilder.build() private val forYouWhoToSubscribeCandidatePipelineConfig = forYouWhoToSubscribeCandidatePipelineConfigBuilder.build() private val flipPromptCandidatePipelineConfig = flipPromptCandidatePipelineConfigBuilder.build[ForYouQuery]( supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) ) private val forYouEventsEntryPointPivotCandidatePipelineConfig = forYouEntryPointPivotCandidatePipelineBuilder.build( entryPointPivotType = ForYouEntryPointPivotCandidatePipelineBuilder.EntryPointPivotType.Events, supportedClientParam = EnableEntryPointPivotParam, pivotMinInjectionIntervalParam = EntryPointPivotMinInjectionIntervalParam ) private val forYouGrokEntryPointPivotCandidatePipelineConfig = forYouEntryPointPivotCandidatePipelineBuilder.build( entryPointPivotType = ForYouEntryPointPivotCandidatePipelineBuilder.EntryPointPivotType.Grok, supportedClientParam = EnableGrokEntryPointPivotParam, pivotMinInjectionIntervalParam = GrokEntryPointPivotMinInjectionIntervalParam ) private val forYouTopicSelectorCandidatePipelineConfig = { jetfuelCandidatePipelineConfigBuilder.build[ForYouQuery]( frameId = "ForYouTopicSelector", identifier = CandidatePipelineIdentifier("ForYouTopicSelector"), route = ForYouTopicSelectorJetfuelRouteParam, gates = Seq( ParamGate(name = "ForYouTopicSelector", param = EnableForYouTopicSelectorParam) ) ) } private val forYouAppUpsellCandidatePipelineConfig = { jetfuelCandidatePipelineConfigBuilder.build[ForYouQuery]( frameId = "ForYouAppUpsell", identifier = CandidatePipelineIdentifier("ForYouAppUpsell"), route = ForYouAppUpsellJetfuelRouteParam, gates = Seq( ParamGate(name = "ForYouAppUpsell", param = EnableForYouAppUpsellParam) ) ) } override val candidatePipelines: Seq[CandidatePipelineConfig[ForYouQuery, _, _, _]] = Seq( forYouScoredTweetsCandidatePipelineConfig, forYouAdsCandidatePipelineConfig, forYouCommunitiesToJoinCandidatePipelineConfig, forYouWhoToFollowCandidatePipelineConfig, forYouWhoToSubscribeCandidatePipelineConfig, forYouTweetPreviewsCandidatePipelineConfig, forYouRecommendedJobsCandidatePipelineConfig, forYouEventsEntryPointPivotCandidatePipelineConfig, forYouGrokEntryPointPivotCandidatePipelineConfig, forYouRecommendedRecruitingOrganizationsCandidatePipelineConfig, forYouBookmarksCandidatePipelineConfig, forYouExplorationTweetsCandidatePipelineConfig, forYouPinnedTweetsCandidatePipelineConfig, forYouStoriesCandidatePipelineConfig, forYouScoredVideoTweetsCandidatePipelineConfig, forYouTuneFeedCandidatePipelineConfig, flipPromptCandidatePipelineConfig, forYouKeywordTrendsCandidatePipelineConfig, forYouJetfuelFrameCandidatePipelineConfig, forYouRelevancePromptCandidatePipelineConfig, forYouTopicSelectorCandidatePipelineConfig, forYouAppUpsellCandidatePipelineConfig, ) override val dependentCandidatePipelines: Seq[ DependentCandidatePipelineConfig[ForYouQuery, _, _, _] ] = Seq( editedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig, verifiedPromptCandidatePipelineConfig ) override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( forYouScoredTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouCommunitiesToJoinCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouWhoToSubscribeCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouTweetPreviewsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouRecommendedJobsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouRecommendedRecruitingOrganizationsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouBookmarksCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouExplorationTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouPinnedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouStoriesCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouScoredVideoTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouTuneFeedCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouKeywordTrendsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouJetfuelFrameCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouEventsEntryPointPivotCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouGrokEntryPointPivotCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouRelevancePromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouTopicSelectorCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouAppUpsellCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, ) override val resultSelectors: Seq[Selector[ForYouQuery]] = Seq( UpdateSortCandidates( ordering = CandidatesUtil.scoreOrdering, candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier ), UpdateSortModuleItemCandidates( candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier, ordering = CandidatesUtil.conversationModuleTweetsOrdering ), UpdateSortCandidates( ordering = CandidatesUtil.scoreOrdering, candidatePipeline = forYouPinnedTweetsCandidatePipelineConfig.identifier ), UpdateSortModuleItemCandidates( ordering = CandidatesUtil.scoreOrdering, candidatePipeline = forYouPinnedTweetsCandidatePipelineConfig.identifier ), UpdateConversationModuleId( pipelineScope = SpecificPipeline(forYouScoredTweetsCandidatePipelineConfig.identifier) ), RemoveDuplicateCandidatesOutsideModule( pipelineScope = SpecificPipeline(forYouScoredVideoTweetsCandidatePipelineConfig.identifier), candidatePipelinesOutsideModule = Set(forYouScoredTweetsCandidatePipelineConfig.identifier), numCandidatesToCompareAgainst = VideoCarouselNumTweetCandidatesToDedupeAgainstParam ), RemoveDuplicateCandidatesOutsideModule( pipelineScope = SpecificPipeline(forYouTuneFeedCandidatePipelineConfig.identifier), candidatePipelinesOutsideModule = Set(forYouScoredTweetsCandidatePipelineConfig.identifier), numCandidatesToCompareAgainst = StaticParam(10) ), DropMaxCandidates( candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier, maxSelectionsParam = ServerMaxResultsParam ), DropMaxCandidates( candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, maxSelectionsParam = MaxNumberReplaceInstructionsParam ), DropMaxCandidates( candidatePipeline = forYouExplorationTweetsCandidatePipelineConfig.identifier, maxSelectionsParam = MaxNumberExplorationTweetsParam ), DropMaxModuleItemCandidates( candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, maxModuleItemsParam = StaticParam(WhoToFollowCandidatePipelineConfig.MaxCandidatesSize) ), DropMaxModuleItemCandidates( candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, maxModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MaxCandidatesSize) ), DropMaxModuleItemCandidates( candidatePipeline = forYouBookmarksCandidatePipelineConfig.identifier, maxModuleItemsParam = StaticParam(10) ), DropMaxModuleItemCandidates( candidatePipeline = forYouScoredVideoTweetsCandidatePipelineConfig.identifier, maxModuleItemsParam = VideoCarouselNumCandidates ), DropMaxModuleItemCandidates( candidatePipeline = forYouPinnedTweetsCandidatePipelineConfig.identifier, maxModuleItemsParam = StaticParam(10) ), DropMaxModuleItemCandidates( candidatePipeline = forYouTuneFeedCandidatePipelineConfig.identifier, maxModuleItemsParam = StaticParam(3) ), DropMaxCandidates( candidatePipeline = forYouPinnedTweetsCandidatePipelineConfig.identifier, maxSelectionsParam = StaticParam(1) ), DropDuplicateCandidates( pipelineScope = SpecificPipelines( Set( forYouScoredTweetsCandidatePipelineConfig.identifier, forYouExplorationTweetsCandidatePipelineConfig.identifier )), duplicationKey = IdAndClassDuplicationKey, mergeStrategy = PickFirstCandidateMerger ), InsertAppendResults( candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier ), DropOrthogonalCandidates( orthogonalCandidatePipelines = Seq( forYouKeywordTrendsCandidatePipelineConfig.identifier, forYouStoriesCandidatePipelineConfig.identifier, forYouWhoToFollowCandidatePipelineConfig.identifier, forYouWhoToSubscribeCandidatePipelineConfig.identifier, forYouTweetPreviewsCandidatePipelineConfig.identifier, forYouRecommendedJobsCandidatePipelineConfig.identifier, forYouRecommendedRecruitingOrganizationsCandidatePipelineConfig.identifier, forYouCommunitiesToJoinCandidatePipelineConfig.identifier, forYouBookmarksCandidatePipelineConfig.identifier, ) ), DropOrthogonalCandidates( orthogonalCandidatePipelines = Seq( forYouEventsEntryPointPivotCandidatePipelineConfig.identifier, forYouGrokEntryPointPivotCandidatePipelineConfig.identifier, ) ), InsertFixedPositionResults( candidatePipeline = forYouAppUpsellCandidatePipelineConfig.identifier, positionParam = ForYouAppUpsellPosition ), InsertFixedPositionResults( candidatePipeline = verifiedPromptCandidatePipelineConfig.identifier, positionParam = StaticParamValueZero ), InsertDynamicPositionResults( candidatePipeline = flipPromptCandidatePipelineConfig.identifier, dynamicInsertionPosition = FlipPromptDynamicInsertionPosition(StaticParamValueZero) ), InsertFixedPositionResults( candidatePipeline = forYouExplorationTweetsCandidatePipelineConfig.identifier, positionParam = ExplorationTweetsTimelinePosition ), InsertFixedPositionResults( candidatePipeline = forYouJetfuelFrameCandidatePipelineConfig.identifier, positionParam = SuperbowlModuleTimelinePosition ), InsertFixedPositionResults( candidatePipeline = forYouEventsEntryPointPivotCandidatePipelineConfig.identifier, positionParam = SuperbowlModuleTimelinePosition ), InsertFixedPositionResults( candidatePipeline = forYouGrokEntryPointPivotCandidatePipelineConfig.identifier, positionParam = GrokPivotModuleTimelinePosition ), InsertFixedPositionResults( candidatePipeline = forYouPinnedTweetsCandidatePipelineConfig.identifier, positionParam = StaticParam(3) ), InsertFixedPositionResults( candidatePipeline = forYouScoredVideoTweetsCandidatePipelineConfig.identifier, positionParam = VideoCarouselTimelinePosition ), InsertFixedPositionResults( candidatePipeline = forYouTuneFeedCandidatePipelineConfig.identifier, positionParam = TuneFeedTimelinePosition ), InsertFixedPositionResults( candidatePipeline = forYouTopicSelectorCandidatePipelineConfig.identifier, positionParam = ForYouTopicSelectorPosition ), InsertFixedPositionResults( candidatePipelines = Set( forYouKeywordTrendsCandidatePipelineConfig.identifier, forYouStoriesCandidatePipelineConfig.identifier, forYouWhoToFollowCandidatePipelineConfig.identifier, forYouWhoToSubscribeCandidatePipelineConfig.identifier, forYouTweetPreviewsCandidatePipelineConfig.identifier, forYouRecommendedJobsCandidatePipelineConfig.identifier, forYouRecommendedRecruitingOrganizationsCandidatePipelineConfig.identifier, forYouCommunitiesToJoinCandidatePipelineConfig.identifier, forYouBookmarksCandidatePipelineConfig.identifier, ), positionParam = StaticParamValueFive ), DropModuleTooFewModuleItemResults( candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, minModuleItemsParam = StaticParam(WhoToFollowCandidatePipelineConfig.MinCandidatesSize) ), DropModuleTooFewModuleItemResults( candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, minModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MinCandidatesSize) ), DropModuleTooFewModuleItemResults( candidatePipeline = forYouBookmarksCandidatePipelineConfig.identifier, minModuleItemsParam = StaticParam(2) ), DropModuleTooFewModuleItemResults( candidatePipeline = forYouScoredVideoTweetsCandidatePipelineConfig.identifier, minModuleItemsParam = StaticParam(3) ), DropModuleTooFewModuleItemResults( candidatePipeline = forYouStoriesCandidatePipelineConfig.identifier, minModuleItemsParam = StaticParam(3) ), DropModuleTooFewModuleItemResults( candidatePipeline = forYouKeywordTrendsCandidatePipelineConfig.identifier, minModuleItemsParam = StaticParam(3) ), SelectConditionally.paramNotGated( InsertAdResults( surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), adsCandidatePipeline = forYouAdsCandidatePipelineConfig.identifier ), EnableForYouTimelineAdsSurface ), SelectConditionally.paramGated( InsertAdResults( surfaceAreaName = AdsInjectionSurfaceAreas.ForYouTimeline, adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.ForYouTimeline), adsCandidatePipeline = forYouAdsCandidatePipelineConfig.identifier ), EnableForYouTimelineAdsSurface ), SelectConditionally( DebugUpdateSortAdsResult(forYouAdsCandidatePipelineConfig.identifier), includeSelector = (query, _, _) => query.params(EnableAdsDebugParam) ), // This selector must come after the tweets are inserted into the results UpdateNewTweetsPillDecoration( pipelineScope = SpecificPipelines( forYouScoredTweetsCandidatePipelineConfig.identifier, newTweetsPillCandidatePipelineConfig.identifier ), stringCenter = stringCenterProvider.get(), seeNewTweetsString = externalStrings.seeNewTweetsString, tweetedString = externalStrings.tweetedString ), InsertAppendResults(candidatePipeline = editedTweetsCandidatePipelineConfig.identifier), SelectConditionally( selector = InsertAppendResults(candidatePipeline = newTweetsPillCandidatePipelineConfig.identifier), includeSelector = (_, _, results) => CandidatesUtil.containsType[TweetCandidate](results) ), InsertFixedPositionResults( candidatePipeline = forYouRelevancePromptCandidatePipelineConfig.identifier, positionParam = RelevancePromptTweetPositionParam ), UpdateHomeClientEventDetails( candidatePipelines = Set( forYouScoredTweetsCandidatePipelineConfig.identifier, forYouTweetPreviewsCandidatePipelineConfig.identifier, forYouExplorationTweetsCandidatePipelineConfig.identifier, forYouBookmarksCandidatePipelineConfig.identifier, forYouPinnedTweetsCandidatePipelineConfig.identifier, forYouScoredVideoTweetsCandidatePipelineConfig.identifier, forYouTuneFeedCandidatePipelineConfig.identifier, ) ) ) private val servedCandidateFeatureKeysKafkaSideEffect = servedCandidateFeatureKeysKafkaSideEffectBuilder.build( Set(forYouScoredTweetsCandidatePipelineConfig.identifier)) private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( enableScribeClientEvents = enableScribeClientEvents, logPipelinePublisher = clientEventsScribeEventPublisher, injectedTweetsCandidatePipelineIdentifiers = Seq(forYouScoredTweetsCandidatePipelineConfig.identifier), adsCandidatePipelineIdentifier = Some(forYouAdsCandidatePipelineConfig.identifier), whoToFollowCandidatePipelineIdentifier = Some(forYouWhoToFollowCandidatePipelineConfig.identifier), whoToSubscribeCandidatePipelineIdentifier = Some(forYouWhoToSubscribeCandidatePipelineConfig.identifier), forYouCommunitiesToJoinCandidatePipelineIdentifier = Some(forYouCommunitiesToJoinCandidatePipelineConfig.identifier), forYouRelevancePromptCandidatePipelineIdentifier = Some(forYouRelevancePromptCandidatePipelineConfig.identifier) ) override val resultSideEffects: Seq[PipelineResultSideEffect[ForYouQuery, Timeline]] = Seq( updateTimelinesPersistenceStoreSideEffect, truncateTimelinesPersistenceStoreSideEffect, homeScribeClientEventSideEffect, homeScribeServedCandidatesSideEffect, servedCandidateFeatureKeysKafkaSideEffect, ServedStatsSideEffect( candidatePipelines = Set(forYouScoredTweetsCandidatePipelineConfig.identifier), statsReceiver = statsReceiver ), VideoServedStatsSideEffect( candidatePipelines = Set(forYouScoredTweetsCandidatePipelineConfig.identifier), statsReceiver = statsReceiver ), ) override val domainMarshaller: DomainMarshaller[ForYouQuery, Timeline] = ForYouResponseDomainMarshaller override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = urtTransportMarshaller } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPinnedTweetsCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.decorator.PinnedTweetBroadcastCandidateDecorator import com.twitter.home_mixer.functional_component.feature_hydrator.GizmoduckAuthorFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.filter.CurrentPinnedTweetFilter import com.twitter.home_mixer.functional_component.filter.TweetHydrationFilter import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.product.for_you.candidate_source.BroadcastedPinnedTweetsCandidateSource import com.twitter.home_mixer.product.for_you.candidate_source.PinnedTweetCandidate import com.twitter.home_mixer.product.for_you.feature_hydrator.CurrentPinnedTweetFeatureHydrator import com.twitter.home_mixer.product.for_you.filter.NotArticleFilter import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnablePinnedTweetsCandidatePipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.PinnedTweetsModuleMinInjectionIntervalParam import com.twitter.home_mixer.product.for_you.response_transformer.PinnedTweetResponseFeatureTransformer import com.twitter.home_mixer.product.for_you.scorer.PinnedTweetCandidateScorer import com.twitter.product_mixer.component_library.filter.FeatureFilter import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelines.configapi.FSParam import com.twitter.timelineservice.model.rich.EntityIdType import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouPinnedTweetsCandidatePipelineConfig @Inject() ( broadcastedPinnedTweetsCandidateSource: BroadcastedPinnedTweetsCandidateSource, currentPinnedTweetFeatureHydrator: CurrentPinnedTweetFeatureHydrator, gizmoduckAuthorFeatureHydrator: GizmoduckAuthorFeatureHydrator, tweetypieFeatureHydrator: TweetypieFeatureHydrator, pinnedTweetBroadcastCandidateDecorator: PinnedTweetBroadcastCandidateDecorator, ) extends CandidatePipelineConfig[ ForYouQuery, ForYouQuery, PinnedTweetCandidate, TweetCandidate ] { private val InNetworkFilterId = "InNetwork" override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouPinnedTweets") override val supportedClientParam: Option[FSParam[Boolean]] = Some( EnablePinnedTweetsCandidatePipelineParam) override val gates: Seq[Gate[PipelineQuery]] = Seq( DefinedUserIdGate, RateLimitGate, TimelinesPersistenceStoreLastInjectionGate( PinnedTweetsModuleMinInjectionIntervalParam, EntityIdType.PinnedTweetsModule ), ) override val queryFeatureHydration: Seq[QueryFeatureHydrator[PipelineQuery]] = Seq() override val queryTransformer: CandidatePipelineQueryTransformer[ ForYouQuery, ForYouQuery ] = identity override def candidateSource: CandidateSource[ForYouQuery, PinnedTweetCandidate] = broadcastedPinnedTweetsCandidateSource override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[PinnedTweetCandidate] ] = Seq(PinnedTweetResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ PinnedTweetCandidate, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] ] = Seq( tweetypieFeatureHydrator, InNetworkFeatureHydrator, currentPinnedTweetFeatureHydrator, gizmoduckAuthorFeatureHydrator ) override val filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( TweetHydrationFilter, NotArticleFilter, FeatureFilter.fromFeature(FilterIdentifier(InNetworkFilterId), InNetworkFeature), CurrentPinnedTweetFilter ) override def scorers: Seq[Scorer[ForYouQuery, TweetCandidate]] = Seq(PinnedTweetCandidateScorer) override val decorator: Option[ CandidateDecorator[PipelineQuery, TweetCandidate] ] = Some(pinnedTweetBroadcastCandidateDecorator) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.ForYouProductContext import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.ServerMaxResultsParam import com.twitter.home_mixer.product.for_you.param.ForYouParamConfig import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer import com.twitter.product_mixer.component_library.premarshaller.cursor.timelines.ChronologicalCursorUnmarshaller import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy import com.twitter.product_mixer.core.functional_component.common.alert.Alert import com.twitter.product_mixer.core.functional_component.common.alert.EmptyResponseRateAlert import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert import com.twitter.product_mixer.core.functional_component.common.alert.P99 import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert import com.twitter.product_mixer.core.functional_component.common.alert.ThroughputAlert import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.request.Product import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor import com.twitter.product_mixer.core.pipeline.PipelineConfig import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig import com.twitter.product_mixer.core.product.ProductParamConfig import com.twitter.product_mixer.core.util.SortIndexBuilder import com.twitter.timelines.configapi.Params import com.twitter.timelines.render.{thriftscala => urt} import com.twitter.timelines.util.RequestCursorSerializer import com.twitter.util.Time import com.twitter.util.Try import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouProductPipelineConfig @Inject() ( forYouMixerPipelineConfig: ForYouMixerPipelineConfig, forYouParamConfig: ForYouParamConfig) extends ProductPipelineConfig[HomeMixerRequest, ForYouQuery, urt.TimelineResponse] { override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("ForYou") override val product: Product = ForYouProduct override val paramConfig: ProductParamConfig = forYouParamConfig override def pipelineQueryTransformer( request: HomeMixerRequest, params: Params ): ForYouQuery = { val context = request.productContext match { case Some(context: ForYouProductContext) => context case _ => throw PipelineFailure(BadRequest, "ForYouProductContext not found") } val debugOptions = request.debugParams.flatMap(_.debugOptions) /** * Unlike other clients, newly created tweets on Android have the sort index set to the current * time instead of the top sort index + 1, so these tweets get stuck at the top of the timeline * if subsequent timeline responses use the sort index from the previous response instead of * the current time. */ val pipelineCursor = request.serializedRequestCursor.flatMap { cursor => Try(UrtCursorSerializer.deserializeOrderedCursor(cursor)) .getOrElse(ChronologicalCursorUnmarshaller(RequestCursorSerializer.deserialize(cursor))) .map { case topCursor @ UrtOrderedCursor(_, _, Some(TopCursor), _) => val queryTime = debugOptions.flatMap(_.requestTimeOverride).getOrElse(Time.now) topCursor.copy(initialSortIndex = SortIndexBuilder.timeToId(queryTime)) case cursor => cursor } } ForYouQuery( params = params, clientContext = request.clientContext, features = None, pipelineCursor = pipelineCursor, requestedMaxResults = Some(params(ServerMaxResultsParam)), debugOptions = debugOptions, deviceContext = context.deviceContext, seenTweetIds = context.seenTweetIds, dspClientContext = context.dspClientContext ) } override val pipelines: Seq[PipelineConfig] = Seq(forYouMixerPipelineConfig) override def pipelineSelector(query: ForYouQuery): ComponentIdentifier = forYouMixerPipelineConfig.identifier override val alerts: Seq[Alert] = Seq( SuccessRateAlert( notificationGroup = DefaultNotificationGroup, warnPredicate = TriggerIfBelow(99.9, 20, 30), criticalPredicate = TriggerIfBelow(99.9, 30, 30), ), LatencyAlert( notificationGroup = DefaultNotificationGroup, percentile = P99, warnPredicate = TriggerIfLatencyAbove(2800.millis, 15, 30), criticalPredicate = TriggerIfLatencyAbove(3000.millis, 15, 30) ), ThroughputAlert( notificationGroup = DefaultNotificationGroup, warnPredicate = TriggerIfAbove(70000), criticalPredicate = TriggerIfAbove(80000) ), EmptyResponseRateAlert( notificationGroup = DefaultNotificationGroup, warnPredicate = TriggerIfAbove(2), criticalPredicate = TriggerIfAbove(3) ) ) override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeMixerPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AddEntriesInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ParamGatedIncludeInstruction import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig import com.twitter.timelines.render.{thriftscala => urt} import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouPushToHomeMixerPipelineConfig @Inject() ( forYouPushToHomeTweetCandidatePipelineConfig: ForYouPushToHomeTweetCandidatePipelineConfig, urtTransportMarshaller: UrtTransportMarshaller) extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse] { override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ForYouPushToHome") override val candidatePipelines: Seq[CandidatePipelineConfig[ForYouQuery, _, _, _]] = Seq(forYouPushToHomeTweetCandidatePipelineConfig) override val resultSelectors: Seq[Selector[ForYouQuery]] = Seq(InsertAppendResults(forYouPushToHomeTweetCandidatePipelineConfig.identifier)) override val domainMarshaller: DomainMarshaller[ForYouQuery, Timeline] = { val instructionBuilders = Seq( ClearCacheInstructionBuilder( ParamGatedIncludeInstruction(ForYouParam.EnableClearCacheOnPushToHome)), AddEntriesInstructionBuilder()) val idSelector: PartialFunction[UniversalNoun[_], Long] = { case item: TweetItem => item.id } val topCursorBuilder = OrderedTopCursorBuilder(idSelector) val bottomCursorBuilder = OrderedBottomCursorBuilder(idSelector) val metadataBuilder = UrtMetadataBuilder( title = None, scribeConfigBuilder = Some( StaticTimelineScribeConfigBuilder( TimelineScribeConfig( page = Some("for_you_push_to_home"), section = None, entityToken = None) ) ) ) UrtDomainMarshaller( instructionBuilders = instructionBuilders, metadataBuilder = Some(metadataBuilder), cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder) ) } override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = urtTransportMarshaller } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeTweetCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.product.for_you.functional_component.gate.PushToHomeRequestGate import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.candidate_source.PassthroughCandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelineservice.suggests.{thriftscala => st} import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouPushToHomeTweetCandidatePipelineConfig @Inject() () extends CandidatePipelineConfig[ ForYouQuery, ForYouQuery, TweetCandidate, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouPushToHomeTweet") override val gates: Seq[Gate[ForYouQuery]] = Seq(PushToHomeRequestGate) override val queryTransformer: CandidatePipelineQueryTransformer[ ForYouQuery, ForYouQuery ] = identity override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[TweetCandidate] ] = Seq(new CandidateFeatureTransformer[TweetCandidate] { override def features: Set[Feature[_, _]] = Set(SuggestTypeFeature) override val identifier: TransformerIdentifier = TransformerIdentifier("ForYouPushToHomeTweet") override def transform(input: TweetCandidate): FeatureMap = FeatureMapBuilder().add(SuggestTypeFeature, Some(st.SuggestType.Magicrec)).build() }) override val resultTransformer: CandidatePipelineResultsTransformer[ TweetCandidate, TweetCandidate ] = identity override val candidateSource: CandidateSource[ ForYouQuery, TweetCandidate ] = PassthroughCandidateSource( CandidateSourceIdentifier("PushToHomeTweet"), { query => query.pushToHomeTweetId.toSeq.map(TweetCandidate(_)) } ) override val decorator: Option[ CandidateDecorator[ForYouQuery, TweetCandidate] ] = { val tweetItemBuilder = TweetCandidateUrtItemBuilder( clientEventInfoBuilder = HomeClientEventInfoBuilder() ) Some(UrtItemCandidateDecorator(tweetItemBuilder)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouRecommendedJobsCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.candidateservice.core_io.FetchJobRecommendationsView import com.twitter.home_mixer.functional_component.decorator.urt.builder.FeedbackStrings import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature import com.twitter.home_mixer.model.HomeFeatures.ViewerHasJobRecommendationsEnabled import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.for_you.candidate_source.RecommendedJobsCandidateSource import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableRecommendedJobsParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.MaxRecommendedJobCandidatesParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.RecommendedJobMinInjectionIntervalParam import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate import com.twitter.product_mixer.component_library.gate.FeatureGate import com.twitter.product_mixer.component_library.model.candidate.JobCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.job.RecommendedJobsCandidateDecorator import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.configapi.FSParam import com.twitter.timelineservice.model.rich.EntityIdType import com.twitter.timelineservice.suggests.{thriftscala => st} import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton class ForYouRecommendedJobsCandidatePipelineConfig @Inject() ( recommendedJobsProductCandidateSource: RecommendedJobsCandidateSource, @ProductScoped stringCenterProvider: Provider[StringCenter], externalStrings: HomeMixerExternalStrings, feedbackStrings: FeedbackStrings) extends CandidatePipelineConfig[ PipelineQuery, StratoKeyView[Long, FetchJobRecommendationsView], Long, JobCandidate ] { private val stringCenter = stringCenterProvider.get() override val supportedClientParam: Option[FSParam[Boolean]] = Some(EnableRecommendedJobsParam) override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouRecommendedJobs") override val gates = Seq( DefinedUserIdGate, FeatureGate.fromFeature(ViewerHasJobRecommendationsEnabled), RateLimitGate, TimelinesPersistenceStoreLastInjectionGate( RecommendedJobMinInjectionIntervalParam, EntityIdType.JobModule ), DismissFatigueGate(st.SuggestType.Job, DismissInfoFeature) ) override val queryTransformer: CandidatePipelineQueryTransformer[ PipelineQuery, StratoKeyView[Long, FetchJobRecommendationsView] ] = { query => StratoKeyView( query.getRequiredUserId, FetchJobRecommendationsView(count = Some(query.params(MaxRecommendedJobCandidatesParam)))) } override val candidateSource: BaseCandidateSource[ StratoKeyView[Long, FetchJobRecommendationsView], Long ] = recommendedJobsProductCandidateSource override val resultTransformer: CandidatePipelineResultsTransformer[ Long, JobCandidate ] = { jobResult => JobCandidate(id = jobResult) } override val decorator: Option[CandidateDecorator[PipelineQuery, JobCandidate]] = { Some( RecommendedJobsCandidateDecorator( stringCenter = stringCenter, headerString = externalStrings.RecommendedJobHeaderString, footerString = externalStrings.RecommendedJobFooterString, seeLessOftenString = feedbackStrings.seeLessOftenFeedbackString, seeLessOftenConfirmationString = feedbackStrings.seeLessOftenConfirmationFeedbackString )) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouRecommendedRecruitingOrganizationsCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.decorator.urt.builder.FeedbackStrings import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature import com.twitter.home_mixer.model.HomeFeatures.ViewerHasRecruitingOrganizationRecommendationsEnabled import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.for_you.candidate_source.RecommendedRecruitingOrganizationsCandidateSource import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableRecommendedRecruitingOrganizationsParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.MaxRecommendedRecruitingOrganizationCandidatesParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.RecommendedRecruitingOrganizationMinInjectionIntervalParam import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate import com.twitter.product_mixer.component_library.gate.FeatureGate import com.twitter.product_mixer.component_library.model.candidate.RecruitingOrganizationCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.recruiting_organization.RecommendedRecruitingOrganizationsCandidateDecorator import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.recruiting.organization.RecruitingOrganizationRecommendationsView import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.configapi.FSParam import com.twitter.timelineservice.model.rich.EntityIdType import com.twitter.timelineservice.suggests.{thriftscala => st} import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton class ForYouRecommendedRecruitingOrganizationsCandidatePipelineConfig @Inject() ( recommendedRecruitingOrganizationsProductCandidateSource: RecommendedRecruitingOrganizationsCandidateSource, @ProductScoped stringCenterProvider: Provider[StringCenter], externalStrings: HomeMixerExternalStrings, feedbackStrings: FeedbackStrings) extends CandidatePipelineConfig[ PipelineQuery, StratoKeyView[Long, RecruitingOrganizationRecommendationsView], Long, RecruitingOrganizationCandidate ] { private val stringCenter = stringCenterProvider.get() override val supportedClientParam: Option[FSParam[Boolean]] = Some(EnableRecommendedRecruitingOrganizationsParam) override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouRecommendedRecruitingOrganizations") override val gates = Seq( DefinedUserIdGate, FeatureGate.fromFeature(ViewerHasRecruitingOrganizationRecommendationsEnabled), RateLimitGate, TimelinesPersistenceStoreLastInjectionGate( RecommendedRecruitingOrganizationMinInjectionIntervalParam, EntityIdType.RecruitingOrganizationModule ), DismissFatigueGate(st.SuggestType.RecruitingOrganization, DismissInfoFeature) ) override val queryTransformer: CandidatePipelineQueryTransformer[ PipelineQuery, StratoKeyView[Long, RecruitingOrganizationRecommendationsView] ] = { query => StratoKeyView( query.getRequiredUserId, RecruitingOrganizationRecommendationsView(count = Some(query.params(MaxRecommendedRecruitingOrganizationCandidatesParam))) ) } override val candidateSource: BaseCandidateSource[ StratoKeyView[Long, RecruitingOrganizationRecommendationsView], Long ] = recommendedRecruitingOrganizationsProductCandidateSource override val resultTransformer: CandidatePipelineResultsTransformer[ Long, RecruitingOrganizationCandidate ] = { orgResult => RecruitingOrganizationCandidate(id = orgResult) } override val decorator: Option[ CandidateDecorator[PipelineQuery, RecruitingOrganizationCandidate] ] = { Some( RecommendedRecruitingOrganizationsCandidateDecorator( stringCenter = stringCenter, headerString = externalStrings.RecommendedRecruitingOrganizationHeaderString, footerString = externalStrings.RecommendedRecruitingOrganizationFooterString, seeLessOftenString = feedbackStrings.seeLessOftenFeedbackString, seeLessOftenConfirmationString = feedbackStrings.seeLessOftenConfirmationFeedbackString )) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouRelevancePromptCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.decorator.urt.builder.RelevancePromptCandidateUrtItemBuilder import com.twitter.home_mixer.functional_component.gate.PersistenceStoreDurationValidationGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.RelevancePromptEnableParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.RelevancePromptMinInjectionIntervalParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.RelevancePromptNegativeParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.RelevancePromptNeutralParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.RelevancePromptPositiveParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.RelevancePromptTitleParam import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.model.candidate.RelevancePromptCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.candidate_source.StaticCandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelines.configapi.FSParam import com.twitter.timelineservice.model.rich.EntityIdType import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouRelevancePromptCandidatePipelineConfig @Inject() () extends CandidatePipelineConfig[ForYouQuery, Unit, Unit, RelevancePromptCandidate] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouRelevancePrompt") override val supportedClientParam: Option[FSParam[Boolean]] = Some(RelevancePromptEnableParam) override val gates = Seq( PersistenceStoreDurationValidationGate(), TimelinesPersistenceStoreLastInjectionGate( RelevancePromptMinInjectionIntervalParam, EntityIdType.Annotation ) ) override val queryTransformer: CandidatePipelineQueryTransformer[ForYouQuery, Unit] = _ => Unit override def candidateSource: CandidateSource[Unit, Unit] = StaticCandidateSource[Unit]( identifier = CandidateSourceIdentifier(identifier.name), result = Seq(Unit) ) override val resultTransformer: CandidatePipelineResultsTransformer[ Unit, RelevancePromptCandidate ] = _ => RelevancePromptCandidate(id = "feed-survey-prompt") override val decorator: Option[CandidateDecorator[ForYouQuery, RelevancePromptCandidate]] = Some( UrtItemCandidateDecorator( RelevancePromptCandidateUrtItemBuilder( RelevancePromptTitleParam, RelevancePromptPositiveParam, RelevancePromptNegativeParam, RelevancePromptNeutralParam )) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouResponseDomainMarshaller.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.decorator.urt.builder.AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder import com.twitter.home_mixer.model.ClearCacheIncludeInstruction import com.twitter.home_mixer.model.NavigationIncludeInstruction import com.twitter.home_mixer.model.request.DeviceContext.RequestContext import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.ClearCache import com.twitter.home_mixer.product.for_you.param.ForYouParam.Navigation import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.NavigationInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedCursorIdSelector.TweetIdSelector import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowCoverInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller import com.twitter.product_mixer.core.model.common.identifier.DomainMarshallerIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig object ForYouResponseDomainMarshaller extends DomainMarshaller[ForYouQuery, Timeline] { override val identifier: DomainMarshallerIdentifier = DomainMarshallerIdentifier("ForYouResponse") override def apply( query: ForYouQuery, selections: Seq[CandidateWithDetails] ): Timeline = { val coldStart = query.deviceContext.flatMap(_.requestContextValue).contains(RequestContext.Launch) val retainViewportItems = if (coldStart && query.params(ClearCache.ColdStartRetainViewportParam)) Some(true) else None val instructionBuilders = Seq( ClearCacheInstructionBuilder( includeInstruction = ClearCacheIncludeInstruction( ClearCache.PtrEnableParam, ClearCache.ColdStartEnableParam, ClearCache.WarmStartEnableParam, ClearCache.ManualRefreshEnableParam, ClearCache.NavigateEnableParam, ClearCache.MinEntriesParam ), retainViewportItems = retainViewportItems ), ReplaceEntryInstructionBuilder(ReplaceAllEntries), // excludes alert, cover, and replace candidates AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder(), ShowAlertInstructionBuilder(), ShowCoverInstructionBuilder(), NavigationInstructionBuilder( NavigationIncludeInstruction( Navigation.PtrEnableParam, Navigation.ColdStartEnableParam, Navigation.WarmStartEnableParam, Navigation.ManualRefreshEnableParam, Navigation.NavigateEnableParam )) ) val topCursorBuilder = OrderedTopCursorBuilder(TweetIdSelector) val bottomCursorBuilder = OrderedBottomCursorBuilder(TweetIdSelector) val scribeConfigBuilder = StaticTimelineScribeConfigBuilder(TimelineScribeConfig(page = Some("for_you"), None, None)) val metadataBuilder = UrtMetadataBuilder(scribeConfigBuilder = Some(scribeConfigBuilder)) val domainMarshaller = UrtDomainMarshaller( instructionBuilders = instructionBuilders, metadataBuilder = Some(metadataBuilder), cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder) ) domainMarshaller(query, selections) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.decorator.ForYouTweetCandidateDecorator import com.twitter.home_mixer.functional_component.feature_hydrator.BasketballContextFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.PostContextFeatureHydrator import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.param.HomeGlobalParams.EnableBasketballContextFeatureHydratorParam import com.twitter.home_mixer.param.HomeGlobalParams.EnablePostContextFeatureHydratorParam import com.twitter.home_mixer.product.for_you.candidate_source.ScoredTweetWithConversationMetadata import com.twitter.home_mixer.product.for_you.candidate_source.ScoredTweetsProductCandidateSource import com.twitter.home_mixer.product.for_you.feature_hydrator.FocalTweetFeatureHydrator import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.feature_hydrator.candidate.communities.CommunityNamesFeatureHydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedBulkCandidateFeatureHydrator import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouScoredTweetsCandidatePipelineConfig @Inject() ( scoredTweetsProductCandidateSource: ScoredTweetsProductCandidateSource, focalTweetFeatureHydrator: FocalTweetFeatureHydrator, namesFeatureHydrator: NamesFeatureHydrator, communityNamesFeatureHydrator: CommunityNamesFeatureHydrator, basketballContextFeatureHydrator: BasketballContextFeatureHydrator, postContextFeatureHydrator: PostContextFeatureHydrator, invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, forYouTweetCandidateDecorator: ForYouTweetCandidateDecorator) extends CandidatePipelineConfig[ ForYouQuery, ForYouQuery, ScoredTweetWithConversationMetadata, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouScoredTweets") override val gates: Seq[Gate[ForYouQuery]] = Seq(RateLimitGate) override val candidateSource: CandidateSource[ForYouQuery, ScoredTweetWithConversationMetadata] = scoredTweetsProductCandidateSource override val queryTransformer: CandidatePipelineQueryTransformer[ForYouQuery, ForYouQuery] = identity override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[ScoredTweetWithConversationMetadata] ] = Seq(ForYouScoredTweetsResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ ScoredTweetWithConversationMetadata, TweetCandidate ] = { sourceResults => TweetCandidate(sourceResults.tweetId) } override val filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( invalidSubscriptionTweetFilter ) override val postFilterFeatureHydration: Seq[ BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] ] = Seq( focalTweetFeatureHydrator, InNetworkFeatureHydrator, namesFeatureHydrator, communityNamesFeatureHydrator, ParamGatedBulkCandidateFeatureHydrator( EnableBasketballContextFeatureHydratorParam, basketballContextFeatureHydrator ), ParamGatedBulkCandidateFeatureHydrator( EnablePostContextFeatureHydratorParam, postContextFeatureHydrator ), ) override val decorator: Option[CandidateDecorator[ForYouQuery, TweetCandidate]] = Some(forYouTweetCandidateDecorator.build()) override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert(10, 20) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsMixerPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.clientapp.{thriftscala => ca} import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig import com.twitter.home_mixer.functional_component.decorator.urt.builder.AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder import com.twitter.home_mixer.functional_component.feature_hydrator._ import com.twitter.home_mixer.functional_component.selector.DebunchCandidates import com.twitter.home_mixer.functional_component.selector.UpdateConversationModuleId import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration import com.twitter.home_mixer.functional_component.side_effect._ import com.twitter.home_mixer.model.ClearCacheIncludeInstruction import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.for_you.feature_hydrator.TimelineServiceTweetsQueryFeatureHydrator import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.ClearCacheOnPtr import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableFlipInjectionModuleCandidatePipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.FlipInlineInjectionModulePosition import com.twitter.home_mixer.product.for_you.param.ForYouParam.ServerMaxResultsParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsPositionParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowPositionParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribePositionParam import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateFeatureKeysKafkaSideEffectBuilder import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateKeysKafkaSideEffectBuilder import com.twitter.home_mixer.product.for_you.side_effect.ServedStatsSideEffect import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptCandidatePipelineConfigBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfig import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowCoverInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder import com.twitter.product_mixer.component_library.selector.DropMaxCandidates import com.twitter.product_mixer.component_library.selector.DropMaxModuleItemCandidates import com.twitter.product_mixer.component_library.selector.DropModuleTooFewModuleItemResults import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.component_library.selector.InsertFixedPositionResults import com.twitter.product_mixer.component_library.selector.SelectConditionally import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates import com.twitter.product_mixer.component_library.selector.UpdateSortModuleItemCandidates import com.twitter.product_mixer.component_library.selector.ads.AdsInjector import com.twitter.product_mixer.component_library.selector.ads.InsertAdResults import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines import com.twitter.product_mixer.core.functional_component.configapi.StaticParam import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem import com.twitter.product_mixer.core.pipeline.FailOpenPolicy import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.render.{thriftscala => urt} import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton class ForYouScoredTweetsMixerPipelineConfig @Inject() ( forYouAdsDependentCandidatePipelineBuilder: ForYouAdsDependentCandidatePipelineBuilder, forYouConversationServiceCandidatePipelineConfig: ForYouConversationServiceCandidatePipelineConfig, forYouPushToHomeTweetCandidatePipelineConfig: ForYouPushToHomeTweetCandidatePipelineConfig, forYouScoredTweetsCandidatePipelineConfig: ForYouScoredTweetsCandidatePipelineConfig, forYouWhoToFollowCandidatePipelineConfigBuilder: ForYouWhoToFollowCandidatePipelineConfigBuilder, forYouWhoToSubscribeCandidatePipelineConfigBuilder: ForYouWhoToSubscribeCandidatePipelineConfigBuilder, flipPromptCandidatePipelineConfigBuilder: FlipPromptCandidatePipelineConfigBuilder, editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[ForYouQuery], forYouTweetPreviewsCandidatePipelineConfig: ForYouTweetPreviewsCandidatePipelineConfig, dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ForYouQuery], timelineServiceTweetsQueryFeatureHydrator: TimelineServiceTweetsQueryFeatureHydrator, previewCreatorsQueryFeatureHydrator: PreviewCreatorsQueryFeatureHydrator, sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, adsInjector: AdsInjector, servedCandidateKeysKafkaSideEffectBuilder: ServedCandidateKeysKafkaSideEffectBuilder, servedCandidateFeatureKeysKafkaSideEffectBuilder: ServedCandidateFeatureKeysKafkaSideEffectBuilder, updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, homeScribeServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, servedStatsSideEffect: ServedStatsSideEffect, clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter], urtTransportMarshaller: UrtTransportMarshaller, @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse] { override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ForYouScoredTweets") private val MaxConsecutiveOutOfNetworkCandidates = 2 private val PushToHomeTweetPosition = 0 private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ForYouQuery]] = Seq( requestQueryFeatureHydrator, persistenceStoreQueryFeatureHydrator, timelineServiceTweetsQueryFeatureHydrator, previewCreatorsQueryFeatureHydrator, sgsFollowedUsersQueryFeatureHydrator, AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), ) private val scoredTweetsCandidatePipelineScope = SpecificPipeline(forYouScoredTweetsCandidatePipelineConfig.identifier) private val forYouAdsCandidatePipelineConfig = forYouAdsDependentCandidatePipelineBuilder .build(scoredTweetsCandidatePipelineScope) private val forYouWhoToFollowCandidatePipelineConfig = forYouWhoToFollowCandidatePipelineConfigBuilder.build() private val forYouWhoToSubscribeCandidatePipelineConfig = forYouWhoToSubscribeCandidatePipelineConfigBuilder.build() private val flipPromptCandidatePipelineConfig = flipPromptCandidatePipelineConfigBuilder.build[ForYouQuery]( supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) ) override val candidatePipelines: Seq[CandidatePipelineConfig[ForYouQuery, _, _, _]] = Seq( forYouScoredTweetsCandidatePipelineConfig, forYouPushToHomeTweetCandidatePipelineConfig, forYouWhoToFollowCandidatePipelineConfig, forYouWhoToSubscribeCandidatePipelineConfig, forYouTweetPreviewsCandidatePipelineConfig, flipPromptCandidatePipelineConfig ) override val dependentCandidatePipelines: Seq[ DependentCandidatePipelineConfig[ForYouQuery, _, _, _] ] = Seq( forYouAdsCandidatePipelineConfig, forYouConversationServiceCandidatePipelineConfig, editedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig ) override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( forYouScoredTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouWhoToSubscribeCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouTweetPreviewsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, ) override val resultSelectors: Seq[Selector[ForYouQuery]] = Seq( UpdateSortCandidates( ordering = CandidatesUtil.reverseChronTweetsOrdering, candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier ), UpdateSortCandidates( ordering = CandidatesUtil.scoreOrdering, candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier ), UpdateSortModuleItemCandidates( candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier, ordering = CandidatesUtil.conversationModuleTweetsOrdering ), DebunchCandidates( pipelineScope = SpecificPipeline(forYouScoredTweetsCandidatePipelineConfig.identifier), mustDebunch = { case item: ItemCandidateWithDetails => !item.features.getOrElse(InNetworkFeature, false) case module: ModuleCandidateWithDetails => !module.candidates.last.features.getOrElse(InNetworkFeature, false) }, maxBunchSize = MaxConsecutiveOutOfNetworkCandidates ), UpdateConversationModuleId( pipelineScope = SpecificPipeline(forYouScoredTweetsCandidatePipelineConfig.identifier) ), DropMaxCandidates( candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier, maxSelectionsParam = ServerMaxResultsParam ), DropMaxCandidates( candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier, maxSelectionsParam = ServerMaxResultsParam ), DropMaxCandidates( candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, maxSelectionsParam = MaxNumberReplaceInstructionsParam ), DropMaxModuleItemCandidates( candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, maxModuleItemsParam = StaticParam(WhoToFollowCandidatePipelineConfig.MaxCandidatesSize) ), DropMaxModuleItemCandidates( candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, maxModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MaxCandidatesSize) ), // The Conversation Service pipeline will only run if the Scored Tweets pipeline returned nothing InsertAppendResults( candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier ), InsertAppendResults( candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier ), InsertFixedPositionResults( candidatePipeline = forYouTweetPreviewsCandidatePipelineConfig.identifier, positionParam = TweetPreviewsPositionParam ), InsertFixedPositionResults( candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, positionParam = WhoToFollowPositionParam ), InsertFixedPositionResults( candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, positionParam = WhoToSubscribePositionParam ), InsertFixedPositionResults( candidatePipeline = flipPromptCandidatePipelineConfig.identifier, positionParam = FlipInlineInjectionModulePosition ), // Insert Push To Home Tweet at top of Timeline InsertFixedPositionResults( candidatePipeline = forYouPushToHomeTweetCandidatePipelineConfig.identifier, positionParam = StaticParam(PushToHomeTweetPosition) ), InsertAdResults( surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), adsCandidatePipeline = forYouAdsCandidatePipelineConfig.identifier ), // This selector must come after the tweets are inserted into the results DropModuleTooFewModuleItemResults( candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, minModuleItemsParam = StaticParam(WhoToFollowCandidatePipelineConfig.MinCandidatesSize) ), DropModuleTooFewModuleItemResults( candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, minModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MinCandidatesSize) ), UpdateNewTweetsPillDecoration( pipelineScope = SpecificPipelines( forYouConversationServiceCandidatePipelineConfig.identifier, forYouScoredTweetsCandidatePipelineConfig.identifier, newTweetsPillCandidatePipelineConfig.identifier ), stringCenter = stringCenterProvider.get(), seeNewTweetsString = externalStrings.seeNewTweetsString, tweetedString = externalStrings.tweetedString ), InsertAppendResults(candidatePipeline = editedTweetsCandidatePipelineConfig.identifier), SelectConditionally( selector = InsertAppendResults(candidatePipeline = newTweetsPillCandidatePipelineConfig.identifier), includeSelector = (_, _, results) => CandidatesUtil.containsType[TweetCandidate](results) ), UpdateHomeClientEventDetails( candidatePipelines = Set( forYouConversationServiceCandidatePipelineConfig.identifier, forYouScoredTweetsCandidatePipelineConfig.identifier ) ), ) private val servedCandidateKeysKafkaSideEffect = servedCandidateKeysKafkaSideEffectBuilder.build( Set(forYouScoredTweetsCandidatePipelineConfig.identifier)) private val servedCandidateFeatureKeysKafkaSideEffect = servedCandidateFeatureKeysKafkaSideEffectBuilder.build( Set(forYouScoredTweetsCandidatePipelineConfig.identifier)) private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( enableScribeClientEvents = enableScribeClientEvents, logPipelinePublisher = clientEventsScribeEventPublisher, injectedTweetsCandidatePipelineIdentifiers = Seq( forYouScoredTweetsCandidatePipelineConfig.identifier, forYouConversationServiceCandidatePipelineConfig.identifier ), adsCandidatePipelineIdentifier = Some(forYouAdsCandidatePipelineConfig.identifier), whoToFollowCandidatePipelineIdentifier = Some( forYouWhoToFollowCandidatePipelineConfig.identifier ), whoToSubscribeCandidatePipelineIdentifier = Some(forYouWhoToSubscribeCandidatePipelineConfig.identifier) ) override val resultSideEffects: Seq[PipelineResultSideEffect[ForYouQuery, Timeline]] = Seq( servedCandidateKeysKafkaSideEffect, servedCandidateFeatureKeysKafkaSideEffect, updateTimelinesPersistenceStoreSideEffect, truncateTimelinesPersistenceStoreSideEffect, homeScribeClientEventSideEffect, homeScribeServedCandidatesSideEffect, servedStatsSideEffect ) override val domainMarshaller: DomainMarshaller[ForYouQuery, Timeline] = { val instructionBuilders = Seq( ClearCacheInstructionBuilder( ClearCacheIncludeInstruction( ClearCacheOnPtr.EnableParam, ClearCacheOnPtr.MinEntriesParam, ) ), ReplaceEntryInstructionBuilder(ReplaceAllEntries), // excludes alert, cover, and replace candidates AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder(), ShowAlertInstructionBuilder(), ShowCoverInstructionBuilder(), ) val idSelector: PartialFunction[UniversalNoun[_], Long] = { // exclude ads while determining tweet cursor values case item: TweetItem if item.promotedMetadata.isEmpty => item.id case module: TimelineModule if module.items.headOption.exists(_.item.isInstanceOf[TweetItem]) => module.items.last.item match { case item: TweetItem => item.id } } val topCursorBuilder = OrderedTopCursorBuilder(idSelector) val bottomCursorBuilder = OrderedBottomCursorBuilder(idSelector) val metadataBuilder = UrtMetadataBuilder( title = None, scribeConfigBuilder = Some( StaticTimelineScribeConfigBuilder( TimelineScribeConfig( page = Some("for_you_scored_tweets"), section = None, entityToken = None))) ) UrtDomainMarshaller( instructionBuilders = instructionBuilders, metadataBuilder = Some(metadataBuilder), cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder) ) } override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = urtTransportMarshaller } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.model.GrokTopics.GrokCategoryIdToNameMap import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.model.PhoenixPredictedBookmarkScoreFeature import com.twitter.home_mixer.model.PhoenixPredictedDwellScoreFeature import com.twitter.home_mixer.model.candidate_source.SourceSignal import com.twitter.home_mixer.model.PhoenixPredictedFavoriteScoreFeature import com.twitter.home_mixer.model.PhoenixPredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature import com.twitter.home_mixer.model.PhoenixPredictedGoodClickConvoDescUamGt2ScoreFeature import com.twitter.home_mixer.model.PhoenixPredictedGoodProfileClickScoreFeature import com.twitter.home_mixer.model.PhoenixPredictedNegativeFeedbackV2ScoreFeature import com.twitter.home_mixer.model.PhoenixPredictedOpenLinkScoreFeature import com.twitter.home_mixer.model.PhoenixPredictedReplyScoreFeature import com.twitter.home_mixer.model.PhoenixPredictedRetweetScoreFeature import com.twitter.home_mixer.model.PhoenixPredictedScreenshotScoreFeature import com.twitter.home_mixer.model.PhoenixPredictedShareScoreFeature import com.twitter.home_mixer.model.PhoenixPredictedVideoQualityViewScoreFeature import com.twitter.home_mixer.model.PredictedDwellScoreFeature import com.twitter.home_mixer.model.PredictedFavoriteScoreFeature import com.twitter.home_mixer.model.PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature import com.twitter.home_mixer.model.PredictedGoodClickConvoDescUamGt2ScoreFeature import com.twitter.home_mixer.model.PredictedGoodProfileClickScoreFeature import com.twitter.home_mixer.model.PredictedNegativeFeedbackV2ScoreFeature import com.twitter.home_mixer.model.PredictedReplyEngagedByAuthorScoreFeature import com.twitter.home_mixer.model.PredictedReplyScoreFeature import com.twitter.home_mixer.model.PredictedRetweetScoreFeature import com.twitter.home_mixer.model.PredictedShareScoreFeature import com.twitter.home_mixer.model.PredictedVideoQualityViewScoreFeature import com.twitter.home_mixer.product.for_you.candidate_source.ScoredTweetWithConversationMetadata import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityNameFeature import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationIdFeature import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecWithEducationTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType import com.twitter.timelines.render.{thriftscala => tl} object ForYouScoredTweetsResponseFeatureTransformer extends CandidateFeatureTransformer[ScoredTweetWithConversationMetadata] { override val identifier: TransformerIdentifier = TransformerIdentifier("ForYouScoredTweetsResponse") override val features: Set[Feature[_, _]] = Set( AncestorsFeature, AuthorIdFeature, AuthorIsBlueVerifiedFeature, AuthorIsCreatorFeature, AuthorIsGoldVerifiedFeature, AuthorIsGrayVerifiedFeature, AuthorIsLegacyVerifiedFeature, AuthorFollowersFeature, ConversationModuleFocalTweetIdFeature, ConversationModuleIdFeature, DirectedAtUserIdFeature, DebugStringFeature, SourceSignalFeature, ExclusiveConversationAuthorIdFeature, FullScoringSucceededFeature, FavoritedByUserIdsFeature, FollowedByUserIdsFeature, InNetworkFeature, InReplyToTweetIdFeature, InReplyToUserIdFeature, IsAncestorCandidateFeature, IsNsfw, IsReadFromCacheFeature, IsRetweetFeature, CommunityIdFeature, CommunityNameFeature, ListIdFeature, ListNameFeature, LocationIdFeature, PredictionRequestIdFeature, QuotedTweetIdFeature, QuotedUserIdFeature, SGSValidFollowedByUserIdsFeature, SGSValidLikedByUserIdsFeature, ValidLikedByUserIdsFeature, ScoreFeature, ServedTypeFeature, SourceTweetIdFeature, SourceUserIdFeature, TopicContextFunctionalityTypeFeature, TopicIdSocialContextFeature, TweetLanguageFeature, TweetTextFeature, TweetTypeMetricsFeature, UserActionsSizeFeature, UserActionsContainsExplicitSignalsFeature, VisibilityReason, ViralContentCreatorFeature, GrokContentCreatorFeature, GorkContentCreatorFeature, HasVideoFeature, VideoDurationMsFeature, TweetMediaIdsFeature, GrokAnnotationsFeature, GrokTopCategoryFeature, GrokIsGoreFeature, GrokIsNsfwFeature, GrokIsSpamFeature, GrokIsViolentFeature, GrokIsLowQualityFeature, GrokIsOcrFeature, PredictedDwellScoreFeature, PredictedFavoriteScoreFeature, PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, PredictedGoodClickConvoDescUamGt2ScoreFeature, PredictedGoodProfileClickScoreFeature, PredictedNegativeFeedbackV2ScoreFeature, PredictedReplyEngagedByAuthorScoreFeature, PredictedReplyScoreFeature, PredictedRetweetScoreFeature, PredictedShareScoreFeature, PredictedVideoQualityViewScoreFeature, PhoenixPredictedDwellScoreFeature, PhoenixPredictedFavoriteScoreFeature, PhoenixPredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, PhoenixPredictedGoodClickConvoDescUamGt2ScoreFeature, PhoenixPredictedGoodProfileClickScoreFeature, PhoenixPredictedNegativeFeedbackV2ScoreFeature, PhoenixPredictedReplyScoreFeature, PhoenixPredictedRetweetScoreFeature, PhoenixPredictedShareScoreFeature, PhoenixPredictedVideoQualityViewScoreFeature, PhoenixPredictedOpenLinkScoreFeature, PhoenixPredictedScreenshotScoreFeature, PhoenixPredictedBookmarkScoreFeature ) override def transform(input: ScoredTweetWithConversationMetadata): FeatureMap = FeatureMapBuilder() .add(AncestorsFeature, input.ancestors.getOrElse(Seq.empty)) .add(AuthorIdFeature, Some(input.authorId)) .add(AuthorIsBlueVerifiedFeature, input.authorIsBlueVerified.getOrElse(false)) .add(AuthorIsGoldVerifiedFeature, input.authorIsGoldVerified.getOrElse(false)) .add(AuthorIsGrayVerifiedFeature, input.authorIsGrayVerified.getOrElse(false)) .add(AuthorIsLegacyVerifiedFeature, input.authorIsLegacyVerified.getOrElse(false)) .add(AuthorIsCreatorFeature, input.authorIsCreator.getOrElse(false)) .add(AuthorFollowersFeature, input.authorFollowers) .add(CommunityIdFeature, input.communityId) .add(CommunityNameFeature, input.communityName) .add(ConversationModuleIdFeature, input.conversationId) .add(ConversationModuleFocalTweetIdFeature, input.conversationFocalTweetId) .add(DirectedAtUserIdFeature, input.directedAtUserId) .add(DebugStringFeature, input.debugString) .add( SourceSignalFeature, input.sourceSignal.map { ss => SourceSignal(ss.id, ss.signalType, ss.signalEntity, ss.authorId) } ) .add(ExclusiveConversationAuthorIdFeature, input.exclusiveConversationAuthorId) .add(SGSValidLikedByUserIdsFeature, input.sgsValidLikedByUserIds.getOrElse(Seq.empty)) .add(SGSValidFollowedByUserIdsFeature, input.sgsValidFollowedByUserIds.getOrElse(Seq.empty)) .add(ValidLikedByUserIdsFeature, input.validLikedByUserIds.getOrElse(Seq.empty)) .add(FavoritedByUserIdsFeature, input.sgsValidLikedByUserIds.getOrElse(Seq.empty)) .add(FollowedByUserIdsFeature, input.sgsValidFollowedByUserIds.getOrElse(Seq.empty)) .add(FullScoringSucceededFeature, true) .add(InNetworkFeature, input.inNetwork.getOrElse(true)) .add(InReplyToTweetIdFeature, input.inReplyToTweetId) .add(InReplyToUserIdFeature, input.inReplyToUserId) .add(IsAncestorCandidateFeature, input.conversationFocalTweetId.exists(_ != input.tweetId)) .add(IsReadFromCacheFeature, input.isReadFromCache.getOrElse(false)) .add(IsRetweetFeature, input.sourceTweetId.isDefined) .add(IsNsfw, input.isNsfw) .add(ListIdFeature, input.listId) .add(ListNameFeature, input.listName) .add(LocationIdFeature, input.locationId) .add(PredictionRequestIdFeature, input.predictionRequestId) .add(QuotedTweetIdFeature, input.quotedTweetId) .add(QuotedUserIdFeature, input.quotedUserId) .add(ScoreFeature, input.score) .add(SourceTweetIdFeature, input.sourceTweetId) .add(SourceUserIdFeature, input.sourceUserId) .add(ServedTypeFeature, input.servedType) .add( TopicContextFunctionalityTypeFeature, input.topicFunctionalityType.collect { case tl.TopicContextFunctionalityType.Basic => BasicTopicContextFunctionalityType case tl.TopicContextFunctionalityType.Recommendation => RecommendationTopicContextFunctionalityType case tl.TopicContextFunctionalityType.RecWithEducation => RecWithEducationTopicContextFunctionalityType } ) .add(TopicIdSocialContextFeature, input.topicId) .add(TweetLanguageFeature, input.tweetLanguage) .add(TweetTextFeature, input.tweetText) .add(TweetTypeMetricsFeature, input.tweetTypeMetrics) .add(UserActionsSizeFeature, input.userActionsSize) .add( UserActionsContainsExplicitSignalsFeature, input.userActionsContainsExplicitSignals.getOrElse(false)) .add(VisibilityReason, input.visibilityReason) .add(ViralContentCreatorFeature, input.viralContentCreatorFeature.getOrElse(false)) .add(GrokContentCreatorFeature, input.grokContentCreatorFeature.getOrElse(false)) .add(GorkContentCreatorFeature, input.gorkContentCreatorFeature.getOrElse(false)) .add(HasVideoFeature, input.hasVideoFeature.getOrElse(false)) .add(VideoDurationMsFeature, input.videoDurationMsFeature) .add(TweetMediaIdsFeature, input.mediaIds.getOrElse(Seq.empty)) .add(GrokAnnotationsFeature, input.grokAnnotations) .add(GrokIsGoreFeature, input.grokAnnotations.flatMap(_.metadata.map(_.isGore))) .add(GrokIsNsfwFeature, input.grokAnnotations.flatMap(_.metadata.map(_.isNsfw))) .add(GrokIsSpamFeature, input.grokAnnotations.flatMap(_.metadata.map(_.isSpam))) .add(GrokIsViolentFeature, input.grokAnnotations.flatMap(_.metadata.map(_.isViolent))) .add(GrokIsLowQualityFeature, input.grokAnnotations.flatMap(_.metadata.map(_.isLowQuality))) .add(GrokIsOcrFeature, input.grokAnnotations.flatMap(_.metadata.map(_.isOcr))) .add( GrokTopCategoryFeature, input.grokAnnotations.flatMap { annotations => annotations.categoryScores.flatMap { scores => val validCategories = scores.collect { case (category, score) if category.forall(_.isDigit) && GrokCategoryIdToNameMap.contains(category.toLong) => (category.toLong, score) } if (validCategories.nonEmpty) Some(validCategories.maxBy(_._2)._1) else None } } ) .add(PredictedDwellScoreFeature, input.predictedScores.flatMap(_.dwellScore)) .add(PredictedFavoriteScoreFeature, input.predictedScores.flatMap(_.favoriteScore)) .add( PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, input.predictedScores.flatMap(_.goodClickConvoDescFavoritedOrRepliedScore)) .add( PredictedGoodClickConvoDescUamGt2ScoreFeature, input.predictedScores.flatMap(_.goodClickConvoDescUamGt2Score)) .add( PredictedGoodProfileClickScoreFeature, input.predictedScores.flatMap(_.goodProfileClickScore)) .add( PredictedNegativeFeedbackV2ScoreFeature, input.predictedScores.flatMap(_.negativeFeedbackV2Score)) .add( PredictedReplyEngagedByAuthorScoreFeature, input.predictedScores.flatMap(_.replyEngagedByAuthorScore)) .add(PredictedReplyScoreFeature, input.predictedScores.flatMap(_.replyScore)) .add(PredictedRetweetScoreFeature, input.predictedScores.flatMap(_.retweetScore)) .add(PredictedShareScoreFeature, input.predictedScores.flatMap(_.shareScore)) .add( PredictedVideoQualityViewScoreFeature, input.predictedScores.flatMap(_.videoQualityViewScore)) .add(PhoenixPredictedDwellScoreFeature, input.phoenixPredictedScores.flatMap(_.dwellScore)) .add( PhoenixPredictedFavoriteScoreFeature, input.phoenixPredictedScores.flatMap(_.favoriteScore)) .add( PhoenixPredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, input.phoenixPredictedScores.flatMap(_.goodClickConvoDescFavoritedOrRepliedScore)) .add( PhoenixPredictedGoodClickConvoDescUamGt2ScoreFeature, input.phoenixPredictedScores.flatMap(_.goodClickConvoDescUamGt2Score)) .add( PhoenixPredictedGoodProfileClickScoreFeature, input.phoenixPredictedScores.flatMap(_.goodProfileClickScore)) .add( PhoenixPredictedNegativeFeedbackV2ScoreFeature, input.phoenixPredictedScores.flatMap(_.negativeFeedbackV2Score)) .add(PhoenixPredictedReplyScoreFeature, input.phoenixPredictedScores.flatMap(_.replyScore)) .add( PhoenixPredictedRetweetScoreFeature, input.phoenixPredictedScores.flatMap(_.retweetScore)) .add(PhoenixPredictedShareScoreFeature, input.phoenixPredictedScores.flatMap(_.shareScore)) .add( PhoenixPredictedVideoQualityViewScoreFeature, input.phoenixPredictedScores.flatMap(_.videoQualityViewScore)) .add( PhoenixPredictedOpenLinkScoreFeature, input.phoenixPredictedScores.flatMap(_.openLinkScore)) .add( PhoenixPredictedScreenshotScoreFeature, input.phoenixPredictedScores.flatMap(_.screenshotScore)) .add( PhoenixPredictedBookmarkScoreFeature, input.phoenixPredictedScores.flatMap(_.bookmarkScore)) .build() } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredVideoTweetsCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.decorator.VideoCarouselModuleCandidateDecorator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.filter.ConsistentAspectRatioFilter import com.twitter.home_mixer.functional_component.filter.TweetHydrationFilter import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.product.for_you.candidate_source.ScoredVideoTweetCandidate import com.twitter.home_mixer.product.for_you.candidate_source.ScoredVideoTweetsCategorizedProductCandidateSource import com.twitter.home_mixer.product.for_you.candidate_source.ScoredVideoTweetsProductCandidateSource import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableScoredVideoTweetsCandidatePipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.VideoCarouselAllowHorizontalVideos import com.twitter.home_mixer.product.for_you.param.ForYouParam.VideoCarouselAllowVerticalVideos import com.twitter.home_mixer.product.for_you.param.ForYouParam.VideoCarouselEnableFooterParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.VideoTweetsModuleMinInjectionIntervalParam import com.twitter.home_mixer.product.for_you.response_transformer.ScoredVideoTweetResponseFeatureTransformer import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelines.configapi.FSParam import com.twitter.timelineservice.model.rich.EntityIdType import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouScoredVideoTweetsCandidatePipelineConfig @Inject() ( scoredVideoTweetsProductCandidateSource: ScoredVideoTweetsProductCandidateSource, scoredVideoTweetsCategorizedProductCandidateSource: ScoredVideoTweetsCategorizedProductCandidateSource, tweetypieFeatureHydrator: TweetypieFeatureHydrator, videoCarouselModuleCandidateDecorator: VideoCarouselModuleCandidateDecorator) extends CandidatePipelineConfig[ ForYouQuery, ForYouQuery, ScoredVideoTweetCandidate, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouScoredVideoTweets") override val supportedClientParam: Option[FSParam[Boolean]] = Some( EnableScoredVideoTweetsCandidatePipelineParam) override val gates: Seq[Gate[PipelineQuery]] = Seq( DefinedUserIdGate, RateLimitGate, TimelinesPersistenceStoreLastInjectionGate( VideoTweetsModuleMinInjectionIntervalParam, EntityIdType.VideoCarouselModule ), ) override val queryFeatureHydration: Seq[QueryFeatureHydrator[PipelineQuery]] = Seq() override val queryTransformer: CandidatePipelineQueryTransformer[ ForYouQuery, ForYouQuery ] = identity override def candidateSource: CandidateSource[ForYouQuery, ScoredVideoTweetCandidate] = scoredVideoTweetsCategorizedProductCandidateSource // scoredVideoTweetsProductCandidateSource override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[ScoredVideoTweetCandidate] ] = Seq(ScoredVideoTweetResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ ScoredVideoTweetCandidate, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] ] = Seq(tweetypieFeatureHydrator) override val filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( TweetHydrationFilter, ConsistentAspectRatioFilter( allowVerticalVideosParam = VideoCarouselAllowVerticalVideos, allowHorizontalVideosParam = VideoCarouselAllowHorizontalVideos) ) override val decorator: Option[ CandidateDecorator[PipelineQuery, TweetCandidate] ] = Some(videoCarouselModuleCandidateDecorator.build(VideoCarouselEnableFooterParam)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouStoriesCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.decorator.StoriesModuleCandidateDecorator import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.product.for_you.candidate_source.StoriesModuleCandidateSource import com.twitter.home_mixer.product.for_you.candidate_source.StoryCandidate import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableTrendsParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.TrendsModuleMinInjectionIntervalParam import com.twitter.home_mixer.product.for_you.response_transformer.StoriesModuleResponseFeatureTransformer import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedTrendCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelines.configapi.FSParam import com.twitter.timelineservice.model.rich.EntityIdType import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouStoriesCandidatePipelineConfig @Inject() ( storiesModule: StoriesModuleCandidateSource, storiesModuleCandidateDecorator: StoriesModuleCandidateDecorator) extends CandidatePipelineConfig[ PipelineQuery, Long, StoryCandidate, UnifiedTrendCandidate ] { override val supportedClientParam: Option[FSParam[Boolean]] = Some(EnableTrendsParam) override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouStories") override val gates = Seq( RateLimitGate, DefinedUserIdGate, TimelinesPersistenceStoreLastInjectionGate( TrendsModuleMinInjectionIntervalParam, EntityIdType.Trends ) ) override val queryTransformer: CandidatePipelineQueryTransformer[ PipelineQuery, Long ] = { query => query.getRequiredUserId } override def candidateSource: BaseCandidateSource[Long, StoryCandidate] = storiesModule override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[StoryCandidate] ] = Seq(StoriesModuleResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ StoryCandidate, UnifiedTrendCandidate ] = { story => UnifiedTrendCandidate(id = story.id.toString) } override val decorator: Option[CandidateDecorator[PipelineQuery, UnifiedTrendCandidate]] = Some(storiesModuleCandidateDecorator.moduleDecorator) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeTweetSocialContextBuilder import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.PerspectiveFilteredSocialContextFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.SGSValidSocialContextFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.filter.FeedbackFatigueFilter import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter import com.twitter.home_mixer.functional_component.filter.RejectTweetFromViewerFilter import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter import com.twitter.home_mixer.functional_component.scorer.FeedbackFatigueScorer import com.twitter.home_mixer.functional_component.scorer.OONTweetScalingScorer import com.twitter.home_mixer.marshaller.timelines.DeviceContextMarshaller import com.twitter.home_mixer.marshaller.timelines.TimelineServiceCursorMarshaller import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature import com.twitter.home_mixer.model.request.DeviceContext import com.twitter.home_mixer.product.for_you.feature_hydrator.FocalTweetFeatureHydrator import com.twitter.home_mixer.product.for_you.filter.SocialContextFilter import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableTimelineScorerCandidatePipelineParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.candidate_source.timeline_scorer.ScoredTweetCandidateWithFocalTweet import com.twitter.product_mixer.component_library.candidate_source.timeline_scorer.TimelineScorerCandidateSource import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ManualModuleId import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder import com.twitter.product_mixer.component_library.filter.FeatureFilter import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.common.alert.Alert import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.model.candidate.CandidateTweetSourceId import com.twitter.timelines.service.{thriftscala => tst} import com.twitter.timelinescorer.{thriftscala => t} import com.twitter.timelineservice.{thriftscala => tlst} import javax.inject.Inject import javax.inject.Singleton /** * Candidate Pipeline Config that fetches tweets from the Timeline Scorer Candidate Source */ @Singleton class ForYouTimelineScorerCandidatePipelineConfig @Inject() ( timelineScorerCandidateSource: TimelineScorerCandidateSource, deviceContextMarshaller: DeviceContextMarshaller, tweetypieFeatureHydrator: TweetypieFeatureHydrator, sgsValidSocialContextFeatureHydrator: SGSValidSocialContextFeatureHydrator, perspectiveFilteredSocialContextFeatureHydrator: PerspectiveFilteredSocialContextFeatureHydrator, namesFeatureHydrator: NamesFeatureHydrator, focalTweetFeatureHydrator: FocalTweetFeatureHydrator, invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, homeTweetSocialContextBuilder: HomeTweetSocialContextBuilder) extends CandidatePipelineConfig[ ForYouQuery, t.ScoredTweetsRequest, ScoredTweetCandidateWithFocalTweet, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouTimelineScorerTweets") private val TweetypieHydratedFilterId = "TweetypieHydrated" private val QuotedTweetDroppedFilterId = "QuotedTweetDropped" private val OutOfNetworkNSFWFilterId = "OutOfNetworkNSFW" private val ConversationModuleNamespace = EntryNamespace("home-conversation") override val supportedClientParam: Option[FSParam[Boolean]] = Some(EnableTimelineScorerCandidatePipelineParam) override val candidateSource: BaseCandidateSource[ t.ScoredTweetsRequest, ScoredTweetCandidateWithFocalTweet ] = timelineScorerCandidateSource override val queryTransformer: CandidatePipelineQueryTransformer[ ForYouQuery, t.ScoredTweetsRequest ] = { query => val deviceContext = query.deviceContext.getOrElse(DeviceContext.Empty) val scoredTweetsRequestContext = t.v1.ScoredTweetsRequestContext( contextualUserId = query.clientContext.userId, timelineId = query.clientContext.userId.map(tlst.TimelineId(tlst.TimelineType.Home, _, None)), deviceContext = Some(deviceContextMarshaller(deviceContext, query.clientContext)), seenTweetIds = query.seenTweetIds, contextualUserContext = Some(tst.ContextualUserContext(query.clientContext.userRoles)), timelineRequestCursor = query.pipelineCursor.flatMap(TimelineServiceCursorMarshaller(_)) ) val candidateTweetSourceIds = Seq( CandidateTweetSourceId.RecycledTweet, CandidateTweetSourceId.OrganicTweet, CandidateTweetSourceId.AncestorsOnlyOrganicTweet, CandidateTweetSourceId.BackfillOrganicTweet, CandidateTweetSourceId.CroonTweet, CandidateTweetSourceId.RecommendedTweet, CandidateTweetSourceId.FrsTweet, CandidateTweetSourceId.ListTweet, CandidateTweetSourceId.RecommendedTrendTweet, CandidateTweetSourceId.PopularTopicTweet ) val timelineServiceTweets = query.features.map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty)).getOrElse(Seq.empty) val timelineEntries = timelineServiceTweets.map { id => tlst.TimelineEntry.Tweet(tlst.Tweet(statusId = id, sortIndex = id)) } t.ScoredTweetsRequest.V1( t.v1.ScoredTweetsRequest( scoredTweetsRequestContext = Some(scoredTweetsRequestContext), candidateTweetSourceIds = Some(candidateTweetSourceIds.flatMap(CandidateTweetSourceId.toThrift)), maxResultsCount = query.requestedMaxResults, organicTimeline = Some( tlst.Timeline( timelineId = tlst.TimelineId( timelineType = tlst.TimelineType.Home, id = query.getRequiredUserId, canonicalTimelineId = None), entries = timelineEntries, modules = tlst.TimelineModules() )) ) ) } override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[ScoredTweetCandidateWithFocalTweet] ] = Seq(ForYouTimelineScorerResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ ScoredTweetCandidateWithFocalTweet, TweetCandidate ] = { candidateWithFocalTweetId => TweetCandidate(id = candidateWithFocalTweetId.candidate.tweetId) } override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] ] = Seq( namesFeatureHydrator, tweetypieFeatureHydrator, InNetworkFeatureHydrator, sgsValidSocialContextFeatureHydrator, perspectiveFilteredSocialContextFeatureHydrator, ) override def filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( RetweetDeduplicationFilter, FeatureFilter.fromFeature(FilterIdentifier(TweetypieHydratedFilterId), IsHydratedFeature), PredicateFeatureFilter.fromPredicate( FilterIdentifier(QuotedTweetDroppedFilterId), shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } ), PredicateFeatureFilter.fromPredicate( FilterIdentifier(OutOfNetworkNSFWFilterId), shouldKeepCandidate = { features => features.getOrElse(InNetworkFeature, false) || !features.getOrElse(IsNsfwFeature, false) } ), FeedbackFatigueFilter, RejectTweetFromViewerFilter, SocialContextFilter, invalidSubscriptionTweetFilter, InvalidConversationModuleFilter ) override val postFilterFeatureHydration: Seq[ BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] ] = Seq(focalTweetFeatureHydrator) override val scorers: Seq[Scorer[ForYouQuery, TweetCandidate]] = Seq(OONTweetScalingScorer, FeedbackFatigueScorer) override val decorator: Option[CandidateDecorator[ForYouQuery, TweetCandidate]] = { val clientEventInfoBuilder = HomeClientEventInfoBuilder() val tweetItemBuilder = TweetCandidateUrtItemBuilder( clientEventInfoBuilder = clientEventInfoBuilder, socialContextBuilder = Some(homeTweetSocialContextBuilder), timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder), feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) ) val tweetDecorator = UrtItemCandidateDecorator(tweetItemBuilder) val moduleBuilder = TimelineModuleBuilder( entryNamespace = ConversationModuleNamespace, clientEventInfoBuilder = clientEventInfoBuilder, moduleIdGeneration = ManualModuleId(0L), displayTypeBuilder = StaticModuleDisplayTypeBuilder(VerticalConversation), metadataBuilder = Some(HomeConversationModuleMetadataBuilder()) ) Some( UrtMultipleModulesDecorator( urtItemCandidateDecorator = tweetDecorator, moduleBuilder = moduleBuilder, groupByKey = (_, _, candidateFeatures) => candidateFeatures.getOrElse(ConversationModuleFocalTweetIdFeature, None) )) } override val alerts: Seq[Alert] = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert(10, 20) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerMixerPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.clientapp.{thriftscala => ca} import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig import com.twitter.home_mixer.functional_component.decorator.urt.builder.AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder import com.twitter.home_mixer.functional_component.feature_hydrator.FeedbackHistoryQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator._ import com.twitter.home_mixer.functional_component.selector.DebunchCandidates import com.twitter.home_mixer.functional_component.selector.UpdateConversationModuleId import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration import com.twitter.home_mixer.functional_component.side_effect._ import com.twitter.home_mixer.model.ClearCacheIncludeInstruction import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.for_you.feature_hydrator.TimelineServiceTweetsQueryFeatureHydrator import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.ClearCacheOnPtr import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableFlipInjectionModuleCandidatePipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.FlipInlineInjectionModulePosition import com.twitter.home_mixer.product.for_you.param.ForYouParam.ServerMaxResultsParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsPositionParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowPositionParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribePositionParam import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateFeatureKeysKafkaSideEffectBuilder import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateKeysKafkaSideEffectBuilder import com.twitter.home_mixer.product.for_you.side_effect.ServedStatsSideEffect import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.AsyncParamGatedQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptCandidatePipelineConfigBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfig import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowCoverInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder import com.twitter.product_mixer.component_library.selector.DropMaxCandidates import com.twitter.product_mixer.component_library.selector.DropMaxModuleItemCandidates import com.twitter.product_mixer.component_library.selector.DropModuleTooFewModuleItemResults import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.component_library.selector.InsertFixedPositionResults import com.twitter.product_mixer.component_library.selector.SelectConditionally import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates import com.twitter.product_mixer.component_library.selector.UpdateSortModuleItemCandidates import com.twitter.product_mixer.component_library.selector.ads.AdsInjector import com.twitter.product_mixer.component_library.selector.ads.InsertAdResults import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines import com.twitter.product_mixer.core.functional_component.configapi.StaticParam import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem import com.twitter.product_mixer.core.pipeline.FailOpenPolicy import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.render.{thriftscala => urt} import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton class ForYouTimelineScorerMixerPipelineConfig @Inject() ( forYouTimelineScorerCandidatePipelineConfig: ForYouTimelineScorerCandidatePipelineConfig, forYouPushToHomeTweetCandidatePipelineConfig: ForYouPushToHomeTweetCandidatePipelineConfig, forYouConversationServiceCandidatePipelineConfig: ForYouConversationServiceCandidatePipelineConfig, forYouAdsDependentCandidatePipelineBuilder: ForYouAdsDependentCandidatePipelineBuilder, forYouWhoToFollowCandidatePipelineConfigBuilder: ForYouWhoToFollowCandidatePipelineConfigBuilder, forYouWhoToSubscribeCandidatePipelineConfigBuilder: ForYouWhoToSubscribeCandidatePipelineConfigBuilder, flipPromptCandidatePipelineConfigBuilder: FlipPromptCandidatePipelineConfigBuilder, editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[ForYouQuery], forYouTweetPreviewsCandidatePipelineConfig: ForYouTweetPreviewsCandidatePipelineConfig, dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, impressionBloomFilterQueryFeatureHydrator: ImpressionBloomFilterQueryFeatureHydrator[ForYouQuery], manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ForYouQuery], memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ForYouQuery], timelineServiceTweetsQueryFeatureHydrator: TimelineServiceTweetsQueryFeatureHydrator, lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator, feedbackHistoryQueryFeatureHydrator: FeedbackHistoryQueryFeatureHydrator, previewCreatorsQueryFeatureHydrator: PreviewCreatorsQueryFeatureHydrator, sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, adsInjector: AdsInjector, servedCandidateKeysKafkaSideEffectBuilder: ServedCandidateKeysKafkaSideEffectBuilder, servedCandidateFeatureKeysKafkaSideEffectBuilder: ServedCandidateFeatureKeysKafkaSideEffectBuilder, updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[ForYouQuery, Timeline], publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, publishImpressionBloomFilterSideEffect: PublishImpressionBloomFilterSideEffect, updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, homeScribeServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, servedStatsSideEffect: ServedStatsSideEffect, clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter], urtTransportMarshaller: UrtTransportMarshaller, @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse] { override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ForYouTimelineScorer") private val MaxConsecutiveOutOfNetworkCandidates = 2 private val PushToHomeTweetPosition = 0 private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep private val resultSelectorsStep = MixerPipelineConfig.resultSelectorsStep override def fetchQueryFeatures: Seq[QueryFeatureHydrator[ForYouQuery]] = Seq( requestQueryFeatureHydrator, persistenceStoreQueryFeatureHydrator, timelineServiceTweetsQueryFeatureHydrator, feedbackHistoryQueryFeatureHydrator, previewCreatorsQueryFeatureHydrator, sgsFollowedUsersQueryFeatureHydrator, AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, lastNonPollingTimeQueryFeatureHydrator), AsyncParamGatedQueryFeatureHydrator( EnableImpressionBloomFilter, resultSelectorsStep, impressionBloomFilterQueryFeatureHydrator), AsyncQueryFeatureHydrator(resultSelectorsStep, manhattanTweetImpressionsQueryFeatureHydrator), AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator) ) private val timelineScorerCandidatePipelineScope = SpecificPipeline(forYouTimelineScorerCandidatePipelineConfig.identifier) private val forYouAdsCandidatePipelineConfig = forYouAdsDependentCandidatePipelineBuilder .build(timelineScorerCandidatePipelineScope) private val forYouWhoToFollowCandidatePipelineConfig = forYouWhoToFollowCandidatePipelineConfigBuilder.build() private val forYouWhoToSubscribeCandidatePipelineConfig = forYouWhoToSubscribeCandidatePipelineConfigBuilder.build() private val flipPromptCandidatePipelineConfig = flipPromptCandidatePipelineConfigBuilder.build[ForYouQuery]( supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) ) override val candidatePipelines: Seq[CandidatePipelineConfig[ForYouQuery, _, _, _]] = Seq( forYouTimelineScorerCandidatePipelineConfig, forYouPushToHomeTweetCandidatePipelineConfig, forYouWhoToFollowCandidatePipelineConfig, forYouWhoToSubscribeCandidatePipelineConfig, forYouTweetPreviewsCandidatePipelineConfig, flipPromptCandidatePipelineConfig ) override val dependentCandidatePipelines: Seq[ DependentCandidatePipelineConfig[ForYouQuery, _, _, _] ] = Seq( forYouAdsCandidatePipelineConfig, forYouConversationServiceCandidatePipelineConfig, editedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig ) override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( forYouTimelineScorerCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouWhoToSubscribeCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, forYouTweetPreviewsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, ) override val resultSelectors: Seq[Selector[ForYouQuery]] = Seq( UpdateSortCandidates( ordering = CandidatesUtil.reverseChronTweetsOrdering, candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier ), UpdateSortCandidates( ordering = CandidatesUtil.scoreOrdering, candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier ), UpdateSortModuleItemCandidates( candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier, ordering = CandidatesUtil.conversationModuleTweetsOrdering ), DebunchCandidates( pipelineScope = SpecificPipeline(forYouTimelineScorerCandidatePipelineConfig.identifier), mustDebunch = { case item: ItemCandidateWithDetails => !item.features.getOrElse(InNetworkFeature, false) case module: ModuleCandidateWithDetails => !module.candidates.last.features.getOrElse(InNetworkFeature, false) }, maxBunchSize = MaxConsecutiveOutOfNetworkCandidates ), UpdateConversationModuleId( pipelineScope = SpecificPipeline(forYouTimelineScorerCandidatePipelineConfig.identifier) ), DropMaxCandidates( candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier, maxSelectionsParam = ServerMaxResultsParam ), DropMaxCandidates( candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier, maxSelectionsParam = ServerMaxResultsParam ), DropMaxCandidates( candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, maxSelectionsParam = MaxNumberReplaceInstructionsParam ), DropMaxModuleItemCandidates( candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, maxModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MaxCandidatesSize) ), DropModuleTooFewModuleItemResults( candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, minModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MinCandidatesSize) ), DropMaxModuleItemCandidates( candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, maxModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MaxCandidatesSize) ), // Add Conversation Service tweets to results only if the scored pipeline doesn't return any SelectConditionally( selector = InsertAppendResults( candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier), includeSelector = (_, candidates, _) => !candidates.exists(candidate => forYouTimelineScorerCandidatePipelineConfig.identifier == candidate.source) ), InsertAppendResults(candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier), InsertFixedPositionResults( candidatePipeline = forYouTweetPreviewsCandidatePipelineConfig.identifier, positionParam = TweetPreviewsPositionParam ), InsertFixedPositionResults( candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, positionParam = WhoToFollowPositionParam ), InsertFixedPositionResults( candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, positionParam = WhoToSubscribePositionParam ), InsertFixedPositionResults( candidatePipeline = flipPromptCandidatePipelineConfig.identifier, positionParam = FlipInlineInjectionModulePosition ), // Insert Push To Home Tweet at top of Timeline InsertFixedPositionResults( candidatePipeline = forYouPushToHomeTweetCandidatePipelineConfig.identifier, positionParam = StaticParam(PushToHomeTweetPosition) ), InsertAdResults( surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), adsCandidatePipeline = forYouAdsCandidatePipelineConfig.identifier ), // This selector must come after the tweets are inserted into the results DropModuleTooFewModuleItemResults( candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, minModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MinCandidatesSize) ), UpdateNewTweetsPillDecoration( pipelineScope = SpecificPipelines( forYouConversationServiceCandidatePipelineConfig.identifier, forYouTimelineScorerCandidatePipelineConfig.identifier, newTweetsPillCandidatePipelineConfig.identifier ), stringCenter = stringCenterProvider.get(), seeNewTweetsString = externalStrings.seeNewTweetsString, tweetedString = externalStrings.tweetedString ), InsertAppendResults(candidatePipeline = editedTweetsCandidatePipelineConfig.identifier), SelectConditionally( selector = InsertAppendResults(candidatePipeline = newTweetsPillCandidatePipelineConfig.identifier), includeSelector = (_, _, results) => CandidatesUtil.containsType[TweetCandidate](results) ), UpdateHomeClientEventDetails( candidatePipelines = Set( forYouConversationServiceCandidatePipelineConfig.identifier, forYouTimelineScorerCandidatePipelineConfig.identifier ) ), ) private val servedCandidateKeysKafkaSideEffect = servedCandidateKeysKafkaSideEffectBuilder.build( Set(forYouTimelineScorerCandidatePipelineConfig.identifier)) private val servedCandidateFeatureKeysKafkaSideEffect = servedCandidateFeatureKeysKafkaSideEffectBuilder.build( Set(forYouTimelineScorerCandidatePipelineConfig.identifier)) private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( enableScribeClientEvents = enableScribeClientEvents, logPipelinePublisher = clientEventsScribeEventPublisher, injectedTweetsCandidatePipelineIdentifiers = Seq( forYouTimelineScorerCandidatePipelineConfig.identifier, forYouConversationServiceCandidatePipelineConfig.identifier ), adsCandidatePipelineIdentifier = Some(forYouAdsCandidatePipelineConfig.identifier), whoToFollowCandidatePipelineIdentifier = Some(forYouWhoToFollowCandidatePipelineConfig.identifier), whoToSubscribeCandidatePipelineIdentifier = Some(forYouWhoToSubscribeCandidatePipelineConfig.identifier) ) override val resultSideEffects: Seq[PipelineResultSideEffect[ForYouQuery, Timeline]] = Seq( homeScribeClientEventSideEffect, homeScribeServedCandidatesSideEffect, publishClientSentImpressionsEventBusSideEffect, publishClientSentImpressionsManhattanSideEffect, publishImpressionBloomFilterSideEffect, servedCandidateKeysKafkaSideEffect, servedCandidateFeatureKeysKafkaSideEffect, servedStatsSideEffect, truncateTimelinesPersistenceStoreSideEffect, updateLastNonPollingTimeSideEffect, updateTimelinesPersistenceStoreSideEffect ) override val domainMarshaller: DomainMarshaller[ForYouQuery, Timeline] = { val instructionBuilders = Seq( ClearCacheInstructionBuilder( ClearCacheIncludeInstruction( ClearCacheOnPtr.EnableParam, ClearCacheOnPtr.MinEntriesParam ) ), ReplaceEntryInstructionBuilder(ReplaceAllEntries), // excludes alert, cover, and replace candidates AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder(), ShowAlertInstructionBuilder(), ShowCoverInstructionBuilder(), ) val idSelector: PartialFunction[UniversalNoun[_], Long] = { // exclude ads while determining tweet cursor values case item: TweetItem if item.promotedMetadata.isEmpty => item.id case module: TimelineModule if module.items.headOption.exists(_.item.isInstanceOf[TweetItem]) => module.items.last.item match { case item: TweetItem => item.id } } val topCursorBuilder = OrderedTopCursorBuilder(idSelector) val bottomCursorBuilder = OrderedBottomCursorBuilder(idSelector) val metadataBuilder = UrtMetadataBuilder( title = None, scribeConfigBuilder = Some( StaticTimelineScribeConfigBuilder( TimelineScribeConfig( page = Some("for_you_timeline_scorer"), section = None, entityToken = None))) ) UrtDomainMarshaller( instructionBuilders = instructionBuilders, metadataBuilder = Some(metadataBuilder), cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder) ) } override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = urtTransportMarshaller } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.mediaservices.commons.tweetmedia.{thriftscala => mt} import com.twitter.product_mixer.component_library.candidate_source.timeline_scorer.ScoredTweetCandidateWithFocalTweet import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecWithEducationTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType import com.twitter.search.common.constants.thriftjava.ThriftLanguage import com.twitter.search.common.util.lang.ThriftLanguageUtil import com.twitter.snowflake.id.SnowflakeId import com.twitter.timelinemixer.injection.model.candidate.AudioSpaceMetaData import com.twitter.timelines.conversation_features.{thriftscala => cvt} import com.twitter.timelinescorer.common.scoredtweetcandidate.{thriftscala => stc} import com.twitter.timelineservice.suggests.{thriftscala => tls} object ForYouTimelineScorerResponseFeatureTransformer extends CandidateFeatureTransformer[ScoredTweetCandidateWithFocalTweet] { override val identifier: TransformerIdentifier = TransformerIdentifier("ForYouTimelineScorerResponse") override val features: Set[Feature[_, _]] = Set( AncestorsFeature, AudioSpaceMetaDataFeature, AuthorIdFeature, AuthorIsBlueVerifiedFeature, AuthorIsCreatorFeature, AuthorIsGoldVerifiedFeature, AuthorIsGrayVerifiedFeature, AuthorIsLegacyVerifiedFeature, AuthoredByContextualUserFeature, CandidateSourceIdFeature, ConversationFeature, ConversationModuleFocalTweetIdFeature, ConversationModuleIdFeature, DirectedAtUserIdFeature, EarlybirdFeature, EntityTokenFeature, ExclusiveConversationAuthorIdFeature, FavoritedByUserIdsFeature, FollowedByUserIdsFeature, TopicIdSocialContextFeature, TopicContextFunctionalityTypeFeature, FromInNetworkSourceFeature, FullScoringSucceededFeature, HasDisplayedTextFeature, InNetworkFeature, InReplyToTweetIdFeature, IsAncestorCandidateFeature, IsExtendedReplyFeature, IsRandomTweetFeature, IsReadFromCacheFeature, IsRetweetFeature, IsRetweetedReplyFeature, NonSelfFavoritedByUserIdsFeature, NumImagesFeature, OriginalTweetCreationTimeFromSnowflakeFeature, PredictionRequestIdFeature, QuotedTweetIdFeature, ScoreFeature, SimclustersTweetTopKClustersWithScoresFeature, SourceTweetIdFeature, SourceUserIdFeature, StreamToKafkaFeature, SuggestTypeFeature, TweetLanguageFeature, VideoDurationMsFeature, ) // Convert language code to ISO 639-3 format private def getLanguageISOFormatByValue(languageCodeValue: Int): String = ThriftLanguageUtil.getLanguageCodeOf(ThriftLanguage.findByValue(languageCodeValue)) override def transform( candidateWithFocalTweet: ScoredTweetCandidateWithFocalTweet ): FeatureMap = { val candidate: stc.v1.ScoredTweetCandidate = candidateWithFocalTweet.candidate val focalTweetId = candidateWithFocalTweet.focalTweetIdOpt val originalTweetId = candidate.sourceTweetId.getOrElse(candidate.tweetId) val tweetFeatures = candidate.tweetFeaturesMap.flatMap(_.get(originalTweetId)) val earlybirdFeatures = tweetFeatures.flatMap(_.recapFeatures.flatMap(_.tweetFeatures)) val directedAtUserIsInFirstDegree = earlybirdFeatures.flatMap(_.directedAtUserIdIsInFirstDegree) val isReply = candidate.inReplyToTweetId.nonEmpty val isRetweet = candidate.isRetweet.getOrElse(false) val isInNetwork = candidate.isInNetwork.getOrElse(true) val conversationFeatures = candidate.conversationFeatures.flatMap { case cvt.ConversationFeatures.V1(candidate) => Some(candidate) case _ => None } val numImages = candidate.mediaMetaData .map( _.count(mediaEntity => mediaEntity.mediaInfo.exists(_.isInstanceOf[mt.MediaInfo.ImageInfo]) || mediaEntity.mediaInfo.isEmpty)) val hasImage = earlybirdFeatures.exists(_.hasImage) val hasVideo = earlybirdFeatures.exists(_.hasVideo) val hasCard = earlybirdFeatures.exists(_.hasCard) val hasQuote = earlybirdFeatures.exists(_.hasQuote.contains(true)) val hasDisplayedText = earlybirdFeatures.exists(_.tweetLength.exists(length => { val numMedia = Seq(hasVideo, (hasImage || hasCard), hasQuote).count(b => b) val tcoLengthsPlusSpaces = 23 * numMedia + (if (numMedia > 0) numMedia - 1 else 0) length > tcoLengthsPlusSpaces })) val suggestType = candidate.overrideSuggestType.orElse(Some(tls.SuggestType.Undefined)) val topicSocialProofMetadataOpt = candidate.entityData.flatMap(_.topicSocialProofMetadata) val topicIdSocialContextOpt = topicSocialProofMetadataOpt.map(_.topicId) val topicContextFunctionalityTypeOpt = topicSocialProofMetadataOpt.map(_.topicContextFunctionalityType).collect { case stc.v1.TopicContextFunctionalityType.Basic => BasicTopicContextFunctionalityType case stc.v1.TopicContextFunctionalityType.Recommendation => RecommendationTopicContextFunctionalityType case stc.v1.TopicContextFunctionalityType.RecWithEducation => RecWithEducationTopicContextFunctionalityType } FeatureMapBuilder() .add( AncestorsFeature, candidate.ancestors .getOrElse(Seq.empty) .map(ancestor => ta.TweetAncestor(ancestor.tweetId, ancestor.userId.getOrElse(0L)))) .add( AudioSpaceMetaDataFeature, candidate.audioSpaceMetaDatalist.map(_.head).map(AudioSpaceMetaData.fromThrift)) .add(AuthorIdFeature, Some(candidate.authorId)) .add(AuthorIsBlueVerifiedFeature, candidate.authorIsBlueVerified.getOrElse(false)) .add( AuthorIsCreatorFeature, candidate.authorIsCreator.getOrElse(false) ) .add(AuthorIsGoldVerifiedFeature, candidate.authorIsGoldVerified.getOrElse(false)) .add(AuthorIsGrayVerifiedFeature, candidate.authorIsGrayVerified.getOrElse(false)) .add(AuthorIsLegacyVerifiedFeature, candidate.authorIsLegacyVerified.getOrElse(false)) .add( AuthoredByContextualUserFeature, candidate.viewerId.contains(candidate.authorId) || candidate.viewerId.exists(candidate.sourceUserId.contains)) .add(CandidateSourceIdFeature, candidate.candidateTweetSourceId) .add(ConversationFeature, conversationFeatures) .add(ConversationModuleIdFeature, candidate.conversationId) .add(ConversationModuleFocalTweetIdFeature, focalTweetId) .add(DirectedAtUserIdFeature, candidate.directedAtUserId) .add(EarlybirdFeature, earlybirdFeatures) // This is temporary, will need to be updated with the encoded string. .add(EntityTokenFeature, Some("test_EntityTokenForYou")) .add(ExclusiveConversationAuthorIdFeature, candidate.exclusiveConversationAuthorId) .add(FavoritedByUserIdsFeature, candidate.favoritedByUserIds.getOrElse(Seq.empty)) .add(FollowedByUserIdsFeature, candidate.followedByUserIds.getOrElse(Seq.empty)) .add(TopicIdSocialContextFeature, topicIdSocialContextOpt) .add(TopicContextFunctionalityTypeFeature, topicContextFunctionalityTypeOpt) .add(FullScoringSucceededFeature, candidate.fullScoringSucceeded.getOrElse(false)) .add(HasDisplayedTextFeature, hasDisplayedText) .add(InNetworkFeature, candidate.isInNetwork.getOrElse(true)) .add(InReplyToTweetIdFeature, candidate.inReplyToTweetId) .add(IsAncestorCandidateFeature, candidate.isAncestorCandidate.getOrElse(false)) .add( IsExtendedReplyFeature, isInNetwork && isReply && !isRetweet && directedAtUserIsInFirstDegree.contains(false)) .add(FromInNetworkSourceFeature, candidate.isInNetwork.getOrElse(true)) .add(IsRandomTweetFeature, candidate.isRandomTweet.getOrElse(false)) .add(IsReadFromCacheFeature, candidate.isReadFromCache.getOrElse(false)) .add(IsRetweetFeature, candidate.isRetweet.getOrElse(false)) .add(IsRetweetedReplyFeature, isReply && isRetweet) .add( NonSelfFavoritedByUserIdsFeature, candidate.favoritedByUserIds.getOrElse(Seq.empty).filterNot(_ == candidate.authorId)) .add(NumImagesFeature, numImages) .add( OriginalTweetCreationTimeFromSnowflakeFeature, SnowflakeId.timeFromIdOpt(originalTweetId)) .add(PredictionRequestIdFeature, candidate.predictionRequestId) .add(ScoreFeature, Some(candidate.score)) .add( SimclustersTweetTopKClustersWithScoresFeature, candidate.simclustersTweetTopKClustersWithScores.map(_.toMap).getOrElse(Map.empty)) .add( StreamToKafkaFeature, candidate.predictionRequestId.nonEmpty && candidate.fullScoringSucceeded.getOrElse(false)) .add(SourceTweetIdFeature, candidate.sourceTweetId) .add(SourceUserIdFeature, candidate.sourceUserId) .add(SuggestTypeFeature, suggestType) .add(QuotedTweetIdFeature, candidate.quotedTweetId) .add( TweetLanguageFeature, earlybirdFeatures.flatMap(_.language.map(_.value)).map(getLanguageISOFormatByValue)) .add(VideoDurationMsFeature, earlybirdFeatures.flatMap(_.videoDurationMs)) .build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTuneFeedCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.geoduck.util.country.CountryInfo import com.twitter.home_mixer.functional_component.decorator.TuneFeedModuleCandidateDecorator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.filter.PreviouslySeenTweetsFilter import com.twitter.home_mixer.functional_component.filter.TweetHydrationFilter import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.model.HomeFeatures.CurrentDisplayedGrokTopicFeature import com.twitter.home_mixer.product.for_you.candidate_source.TuneFeedCandidateSource import com.twitter.home_mixer.product.for_you.candidate_source.TuneFeedRequest import com.twitter.home_mixer.product.for_you.gate.TuneFeedModuleGate import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableTuneFeedCandidatePipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.TuneFeedModuleMinInjectionIntervalParam import com.twitter.home_mixer.product.for_you.response_transformer.TuneFeedFeatureTransformer import com.twitter.product_mixer.component_library.gate.DefinedUserIdGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelines.configapi.FSParam import com.twitter.timelineservice.model.rich.EntityIdType import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouTuneFeedCandidatePipelineConfig @Inject() ( tuneFeedCandidateSource: TuneFeedCandidateSource, tweetypieFeatureHydrator: TweetypieFeatureHydrator, tuneFeedModuleDecorator: TuneFeedModuleCandidateDecorator) extends CandidatePipelineConfig[ ForYouQuery, TuneFeedRequest, TweetCandidate, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouTuneFeed") override val supportedClientParam: Option[FSParam[Boolean]] = Some( EnableTuneFeedCandidatePipelineParam) override val gates: Seq[Gate[ForYouQuery]] = Seq( DefinedUserIdGate, RateLimitGate, TuneFeedModuleGate, TimelinesPersistenceStoreLastInjectionGate( TuneFeedModuleMinInjectionIntervalParam, EntityIdType.TuneFeedModule ), ) override val queryTransformer: CandidatePipelineQueryTransformer[ ForYouQuery, TuneFeedRequest ] = { query => // Safe .get, enforced in TuneFeedModuleGate val displayedGrokTopic = query.features.get.getOrElse(CurrentDisplayedGrokTopicFeature, None).get TuneFeedRequest( grokTopic = displayedGrokTopic._1, language = query.getLanguageCode, placeId = query.getCountryCode.flatMap(CountryInfo.lookupByCode).map(_.placeIdLong) ) } override def candidateSource: BaseCandidateSource[TuneFeedRequest, TweetCandidate] = tuneFeedCandidateSource override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[TweetCandidate] ] = Seq(TuneFeedFeatureTransformer) override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] ] = Seq(tweetypieFeatureHydrator) override val filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( TweetHydrationFilter, PreviouslySeenTweetsFilter ) override val resultTransformer: CandidatePipelineResultsTransformer[ TweetCandidate, TweetCandidate ] = identity override val decorator: Option[CandidateDecorator[ForYouQuery, TweetCandidate]] = Some( tuneFeedModuleDecorator) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.filter.DropMaxCandidatesFilter import com.twitter.home_mixer.functional_component.filter.PreviouslyServedTweetPreviewsFilter import com.twitter.home_mixer.functional_component.filter.TweetHydrationFilter import com.twitter.home_mixer.functional_component.gate.PersistenceStoreDurationValidationGate import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.model.HomeFeatures.AuthorEnabledPreviewsFeature import com.twitter.home_mixer.product.for_you.feature_hydrator.ArticlePreviewTextFeatureHydrator import com.twitter.home_mixer.product.for_you.feature_hydrator.AuthorEnabledPreviewsFeatureHydrator import com.twitter.home_mixer.product.for_you.feature_hydrator.TweetPreviewTweetypieCandidateFeatureHydrator import com.twitter.home_mixer.product.for_you.filter.TweetPreviewTextFilter import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableArticlePreviewTextHydrationParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableTweetPreviewsCandidatePipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsMaxCandidatesParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsMinInjectionIntervalParam import com.twitter.home_mixer.product.for_you.query_transformer.TweetPreviewsQueryTransformer import com.twitter.home_mixer.product.for_you.response_transformer.TweetPreviewResponseFeatureTransformer import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.candidate_source.earlybird.EarlybirdTweetCandidateSource import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsFeature import com.twitter.product_mixer.component_library.filter.FeatureFilter import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures import com.twitter.product_mixer.core.functional_component.common.alert.Alert import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomeTweetPreviewHydrationSafetyLevel import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.timelines.configapi.FSParam import com.twitter.timelineservice.model.rich.EntityIdType import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouTweetPreviewsCandidatePipelineConfig @Inject() ( earlybirdTweetCandidateSource: EarlybirdTweetCandidateSource, authorEnabledPreviewsFeatureHydrator: AuthorEnabledPreviewsFeatureHydrator, tweetPreviewsQueryTransformer: TweetPreviewsQueryTransformer, tweetPreviewTweetypieCandidateFeatureHydrator: TweetPreviewTweetypieCandidateFeatureHydrator, articlePreviewTextFeatureHydrator: ArticlePreviewTextFeatureHydrator, homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) extends CandidatePipelineConfig[ ForYouQuery, eb.EarlybirdRequest, eb.ThriftSearchResult, TweetCandidate ] { val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouTweetPreviews") override val supportedClientParam: Option[FSParam[Boolean]] = Some(EnableTweetPreviewsCandidatePipelineParam) override val gates: Seq[Gate[ForYouQuery]] = { Seq( RateLimitGate, PersistenceStoreDurationValidationGate(), TimelinesPersistenceStoreLastInjectionGate( TweetPreviewsMinInjectionIntervalParam, EntityIdType.TweetPreview ), NonEmptySeqFeatureGate(PreviewCreatorsFeature) ) } override val queryTransformer: CandidatePipelineQueryTransformer[ PipelineQuery, eb.EarlybirdRequest ] = tweetPreviewsQueryTransformer override val candidateSource: CandidateSourceWithExtractedFeatures[ eb.EarlybirdRequest, eb.ThriftSearchResult ] = earlybirdTweetCandidateSource override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[eb.ThriftSearchResult] ] = Seq(TweetPreviewResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ eb.ThriftSearchResult, TweetCandidate ] = { tweet => TweetCandidate(tweet.id) } override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] ] = Seq( authorEnabledPreviewsFeatureHydrator, tweetPreviewTweetypieCandidateFeatureHydrator, ) override val preFilterFeatureHydrationPhase2: Seq[ BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] ] = Seq( ParamGatedCandidateFeatureHydrator( candidateFeatureHydrator = articlePreviewTextFeatureHydrator, enabledParam = EnableArticlePreviewTextHydrationParam)) override val filters: Seq[ Filter[ForYouQuery, TweetCandidate] ] = Seq( PreviouslyServedTweetPreviewsFilter, TweetHydrationFilter, FeatureFilter .fromFeature(FilterIdentifier("AuthorEnabledPreviews"), AuthorEnabledPreviewsFeature), TweetPreviewTextFilter, // NOTE: since earlybird returns candidates sorted by score, this will filter // for the top scoring candidates. Should be updated in the future to not assume order of // candidates from earlybird. DropMaxCandidatesFilter(TweetPreviewsMaxCandidatesParam) ) override val decorator: Option[CandidateDecorator[PipelineQuery, TweetCandidate]] = { val clientEventInfoBuilder = ClientEventInfoBuilder[PipelineQuery, TweetCandidate]( hmt.ServedType.ForYouTweetPreview.originalName) val tweetItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( clientEventInfoBuilder = clientEventInfoBuilder, contextualTweetRefBuilder = Some( ContextualTweetRefBuilder( TweetHydrationContext( safetyLevelOverride = Some(TimelineHomeTweetPreviewHydrationSafetyLevel), outerTweetContext = None ) ) ), feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder), ) Some(UrtItemCandidateDecorator(tweetItemBuilder)) } override val alerts: Seq[Alert] = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(95)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeWhoToFollowFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.functional_component.gate.TestUserProbabilisticGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableWhoToFollowCandidatePipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowDisplayLocationParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowDisplayTypeIdParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowMinInjectionIntervalParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowUserDisplayTypeIdParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ParamWhoToFollowModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.ParamWhoToFollowUserDisplayTypeBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfigBuilder import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.timelineservice.model.rich.EntityIdType import com.twitter.timelineservice.suggests.thriftscala.SuggestType import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouWhoToFollowCandidatePipelineConfigBuilder @Inject() ( whoToFollowArmCandidatePipelineConfigBuilder: WhoToFollowArmCandidatePipelineConfigBuilder, homeWhoToFollowFeedbackActionInfoBuilder: HomeWhoToFollowFeedbackActionInfoBuilder, testUserProbabilisticGate: TestUserProbabilisticGate) { def build(): WhoToFollowArmCandidatePipelineConfig[ForYouQuery] = { val gates: Seq[Gate[ForYouQuery]] = Seq( RateLimitGate, TimelinesPersistenceStoreLastInjectionGate( WhoToFollowMinInjectionIntervalParam, EntityIdType.WhoToFollow ), DismissFatigueGate(SuggestType.WhoToFollow, DismissInfoFeature), testUserProbabilisticGate ) whoToFollowArmCandidatePipelineConfigBuilder.build[ForYouQuery]( identifier = WhoToFollowArmCandidatePipelineConfig.identifier, supportedClientParam = Some(EnableWhoToFollowCandidatePipelineParam), alerts = alerts, gates = gates, moduleDisplayTypeBuilder = ParamWhoToFollowModuleDisplayTypeBuilder(WhoToFollowDisplayTypeIdParam), feedbackActionInfoBuilder = Some(homeWhoToFollowFeedbackActionInfoBuilder), excludedUserIdsFeature = Some(WhoToFollowExcludedUserIdsFeature), userDisplayTypeBuilder = ParamWhoToFollowUserDisplayTypeBuilder(WhoToFollowUserDisplayTypeIdParam), displayLocationParam = WhoToFollowDisplayLocationParam ) } private val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(70), HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.scala ================================================ package com.twitter.home_mixer.product.for_you import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeWhoToSubscribeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate import com.twitter.home_mixer.functional_component.gate.RateLimitGate import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableWhoToSubscribeCandidatePipelineParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribeDisplayTypeIdParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribeMinInjectionIntervalParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ParamWhoToFollowModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfigBuilder import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.timelineservice.model.rich.EntityIdType import com.twitter.timelineservice.suggests.thriftscala.SuggestType import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouWhoToSubscribeCandidatePipelineConfigBuilder @Inject() ( whoToSubscribeCandidatePipelineConfigBuilder: WhoToSubscribeCandidatePipelineConfigBuilder, homeWhoToSubscribeFeedbackActionInfoBuilder: HomeWhoToSubscribeFeedbackActionInfoBuilder) { def build(): WhoToSubscribeCandidatePipelineConfig[ForYouQuery] = { val gates: Seq[Gate[ForYouQuery]] = Seq( RateLimitGate, TimelinesPersistenceStoreLastInjectionGate( WhoToSubscribeMinInjectionIntervalParam, EntityIdType.WhoToSubscribe ), DismissFatigueGate(SuggestType.WhoToSubscribe, DismissInfoFeature) ) whoToSubscribeCandidatePipelineConfigBuilder.build[ForYouQuery]( identifier = WhoToSubscribeCandidatePipelineConfig.identifier, supportedClientParam = Some(EnableWhoToSubscribeCandidatePipelineParam), alerts = alerts, gates = gates, moduleDisplayTypeBuilder = ParamWhoToFollowModuleDisplayTypeBuilder(WhoToSubscribeDisplayTypeIdParam), feedbackActionInfoBuilder = Some(homeWhoToSubscribeFeedbackActionInfoBuilder) ) } private val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(70), HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "events-recos/events-recos-service/src/main/thrift:events-recos-thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", "recruiting/candidate-service/src/main/thrift:thrift-scala", "recruiting/src/main/thrift:thrift-scala", "src/thrift/com/twitter/frigate/bookmarks:bookmarks-thrift-scala", "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", "stitch/stitch-timelineservice/src/main/scala", "strato/config/columns/events/entryPoint:entryPoint-strato-client", "strato/config/columns/frigate/bookmarks:bookmarks-strato-client", "strato/config/columns/recruiting/api/user:user-strato-client", "strato/config/columns/recruiting/candidate_service:candidate_service-strato-client", "strato/config/columns/trends/trip:trip-strato-client", "strato/config/columns/trendsai/ordered:ordered-strato-client", "strato/config/src/thrift/com/twitter/strato/columns/jetfuel:jetfuel-scala", "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", ], exports = [ "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BookmarksCandidateSource.scala ================================================ package com.twitter.home_mixer.product.for_you.candidate_source import com.twitter.frigate.bookmarks.thriftscala.BookmarkedTweet import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.frigate.bookmarks.WeeklyBookmarksOnUserClientColumn import javax.inject.Inject import javax.inject.Singleton import scala.util.Random @Singleton class BookmarksCandidateSource @Inject() ( weeklyBookmarksOnUserClientColumn: WeeklyBookmarksOnUserClientColumn) extends CandidateSource[ Long, BookmarkedTweet ] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("Bookmarks") private val MaxCandidates = 20 private val fetcher = weeklyBookmarksOnUserClientColumn.fetcher // Remove duplicates and unbookmarked tweets private def filterValidBookmarks(bookmarks: Seq[BookmarkedTweet]): Seq[BookmarkedTweet] = { bookmarks .groupBy(_.tweetId) .collect { case (_, duplicatedBookmarks) if !duplicatedBookmarks.exists(_.unbookmark.getOrElse(false)) => duplicatedBookmarks.head }.toSeq } override def apply(userId: Long): Stitch[Seq[BookmarkedTweet]] = { fetcher.fetch(userId, ()).map { res => res.v .map { bookmarks => val filteredBookmarks = filterValidBookmarks(bookmarks.bookmarkedTweets) Random.shuffle(filteredBookmarks).take(MaxCandidates) }.getOrElse(Seq.empty) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BroadcastedPinnedTweetsCandidateSource.scala ================================================ package com.twitter.home_mixer.product.for_you.candidate_source import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import com.twitter.stitch.timelineservice.TimelineService import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilterItem import com.twitter.timelineservice.{thriftscala => t} import javax.inject.Inject import javax.inject.Singleton case class PinnedTweetCandidate(tweetId: Long, userId: Option[Long]) @Singleton class BroadcastedPinnedTweetsCandidateSource @Inject() ( timelineService: TimelineService) extends CandidateSource[ ForYouQuery, PinnedTweetCandidate ] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("BroadcastedPinnedTweets") private val MaxTimelineServiceTweets = 100 private val MaxTweetsForHydration = 20 override def apply(query: ForYouQuery): Stitch[Seq[PinnedTweetCandidate]] = { val userId = query.getRequiredUserId val timelineQueryOptions = t.TimelineQueryOptions( contextualUserId = Some(userId) ) val timelineServiceQuery = t.TimelineQuery( timelineType = t.TimelineType.PinnedTweets, timelineId = userId, maxCount = MaxTimelineServiceTweets.toShort, cursor2 = None, options = Some(timelineQueryOptions), timelineId2 = Some(t.TimelineId(t.TimelineType.PinnedTweets, userId)), ) timelineService .getTimeline(timelineServiceQuery).map { timeline => timeline.entries.collect { case t.TimelineEntry.Tweet(tweet) => PinnedTweetCandidate(tweet.statusId, tweet.userId) } }.map { candidates => val bloomFilterSeq = query.features.map(_.get(ImpressionBloomFilterFeature)).get val bloomFilters = bloomFilterSeq.entries.map(ImpressionBloomFilterItem.fromThrift(_).bloomFilter) val seenTweetIds = query.seenTweetIds.getOrElse(Seq.empty).toSet candidates .filter { pinnedTweetCandidate => !seenTweetIds.contains(pinnedTweetCandidate.tweetId) && !bloomFilters.exists(filter => filter.mayContain(pinnedTweetCandidate.tweetId)) } .take(MaxTweetsForHydration) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/JetfuelFrameCandidateSource.scala ================================================ package com.twitter.home_mixer.product.for_you.candidate_source import com.twitter.strato.columns.jetfuel.thriftscala.JetfuelRouteData import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyViewFetcherSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.strato.client.Fetcher import com.twitter.strato.generated.client.events.entryPoint.JetfuelEntryPointByCountryClientColumn import javax.inject.Inject import javax.inject.Singleton /** * Strato column to fetch entry point for sports to display in search results. */ @Singleton class JetfuelFrameCandidateSource @Inject() ( entryPointClientColumn: JetfuelEntryPointByCountryClientColumn) extends StratoKeyViewFetcherSource[ JetfuelEntryPointByCountryClientColumn.Key, JetfuelEntryPointByCountryClientColumn.View, JetfuelEntryPointByCountryClientColumn.Value, JetfuelRouteData ] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("JetfuelFrame") override val fetcher: Fetcher[ JetfuelEntryPointByCountryClientColumn.Key, JetfuelEntryPointByCountryClientColumn.View, JetfuelEntryPointByCountryClientColumn.Value, ] = entryPointClientColumn.fetcher override def stratoResultTransformer( stratoKey: JetfuelEntryPointByCountryClientColumn.Key, stratoResult: JetfuelEntryPointByCountryClientColumn.Value ): Seq[JetfuelRouteData] = stratoResult } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/RecommendedJobsCandidateSource.scala ================================================ package com.twitter.home_mixer.product.for_you.candidate_source import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyViewFetcherSource import com.twitter.strato.generated.client.recruiting.candidate_service.JobRecommendationCandidatesOnUserClientColumn import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.candidateservice.core_io.FetchJobRecommendationsView import com.twitter.strato.client.Client import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class RecommendedJobsCandidateSource @Inject() ( @Named(BatchedStratoClientWithModerateTimeout) stratoClient: Client) extends StratoKeyViewFetcherSource[ Long, FetchJobRecommendationsView, JobRecommendationCandidatesOnUserClientColumn.Value, Long ] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("RecommendedJobs") val fetcher = stratoClient.fetcher[ Long, FetchJobRecommendationsView, JobRecommendationCandidatesOnUserClientColumn.Value ](JobRecommendationCandidatesOnUserClientColumn.Path) override protected def stratoResultTransformer( stratoKey: Long, stratoResult: JobRecommendationCandidatesOnUserClientColumn.Value ): Seq[Long] = stratoResult.map(_.apiJob) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/RecommendedRecruitingOrganizationsCandidateSource.scala ================================================ package com.twitter.home_mixer.product.for_you.candidate_source import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyViewFetcherSource import com.twitter.strato.generated.client.recruiting.api.user.RecruitingOrganizationRecommendationsOnUserClientColumn import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.recruiting.organization.RecruitingOrganizationRecommendationsView import com.twitter.strato.client.Client import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class RecommendedRecruitingOrganizationsCandidateSource @Inject() ( @Named(BatchedStratoClientWithModerateTimeout) stratoClient: Client) extends StratoKeyViewFetcherSource[ Long, RecruitingOrganizationRecommendationsView, RecruitingOrganizationRecommendationsOnUserClientColumn.Value, Long ] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("RecommendedRecruitingOrganizations") val fetcher = stratoClient.fetcher[ Long, RecruitingOrganizationRecommendationsView, RecruitingOrganizationRecommendationsOnUserClientColumn.Value ](RecruitingOrganizationRecommendationsOnUserClientColumn.Path) override protected def stratoResultTransformer( stratoKey: Long, stratoResult: RecruitingOrganizationRecommendationsOnUserClientColumn.Value ): Seq[Long] = stratoResult.map(_.apiRecruitingOrganization) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.scala ================================================ package com.twitter.home_mixer.product.for_you.candidate_source import com.google.inject.Provider import com.twitter.home_mixer.model.HomeFeatures.ServedAuthorIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SignupCountryFeature import com.twitter.home_mixer.model.HomeFeatures.SignupSourceFeature import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature import com.twitter.home_mixer.model.HomeFeatures.UserFollowersCountFeature import com.twitter.home_mixer.model.HomeFeatures.ViewerAllowsForYouRecommendationsFeature import com.twitter.home_mixer.model.HomeFeatures.ViewerHasPremiumTier import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.model.request.ScoredTweetsProduct import com.twitter.home_mixer.model.request.ScoredTweetsProductContext import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer import com.twitter.product_mixer.core.functional_component.candidate_source.product_pipeline.ProductPipelineCandidateSource import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry import com.twitter.snowflake.id.SnowflakeId import com.twitter.spam.rtf.{thriftscala => spam} import com.twitter.timelines.render.{thriftscala => tl} import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} import javax.inject.Inject import javax.inject.Singleton /** * [[ScoredTweetWithConversationMetadata]] **/ case class ScoredTweetWithConversationMetadata( tweetId: Long, authorId: Long, score: Option[Double] = None, servedType: t.ServedType, sourceTweetId: Option[Long] = None, sourceUserId: Option[Long] = None, quotedTweetId: Option[Long] = None, quotedUserId: Option[Long] = None, inReplyToTweetId: Option[Long] = None, inReplyToUserId: Option[Long] = None, directedAtUserId: Option[Long] = None, inNetwork: Option[Boolean] = None, sgsValidLikedByUserIds: Option[Seq[Long]] = None, sgsValidFollowedByUserIds: Option[Seq[Long]] = None, validLikedByUserIds: Option[Seq[Long]] = None, ancestors: Option[Seq[ta.TweetAncestor]] = None, topicId: Option[Long] = None, topicFunctionalityType: Option[tl.TopicContextFunctionalityType] = None, conversationId: Option[Long] = None, conversationFocalTweetId: Option[Long] = None, isReadFromCache: Option[Boolean] = None, exclusiveConversationAuthorId: Option[Long] = None, authorIsBlueVerified: Option[Boolean] = None, authorIsGoldVerified: Option[Boolean] = None, authorIsGrayVerified: Option[Boolean] = None, authorIsLegacyVerified: Option[Boolean] = None, authorIsCreator: Option[Boolean] = None, authorFollowers: Option[Long] = None, locationId: Option[String] = None, predictionRequestId: Option[Long] = None, communityId: Option[Long] = None, communityName: Option[String] = None, listId: Option[Long] = None, listName: Option[String] = None, isNsfw: Option[Boolean] = None, visibilityReason: Option[spam.FilteredReason] = None, tweetLanguage: Option[String] = None, tweetText: Option[String] = None, tweetTypeMetrics: Option[Seq[Byte]] = None, debugString: Option[String] = None, viralContentCreatorFeature: Option[Boolean] = None, grokContentCreatorFeature: Option[Boolean] = None, gorkContentCreatorFeature: Option[Boolean] = None, hasVideoFeature: Option[Boolean] = None, videoDurationMsFeature: Option[Int] = None, mediaIds: Option[Seq[Long]] = None, grokAnnotations: Option[t.GrokAnnotations] = None, predictedScores: Option[t.PredictedScores] = None, phoenixPredictedScores: Option[t.PredictedScores] = None, sourceSignal: Option[t.SourceSignal] = None, userActionsSize: Option[Int] = None, userActionsContainsExplicitSignals: Option[Boolean] = None) @Singleton class ScoredTweetsProductCandidateSource @Inject() ( override val productPipelineRegistry: Provider[ProductPipelineRegistry], override val paramsBuilder: Provider[ParamsBuilder]) extends ProductPipelineCandidateSource[ ForYouQuery, HomeMixerRequest, t.ScoredTweetsResponse, ScoredTweetWithConversationMetadata ] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("ScoredTweetsProduct") private val MaxModuleSize = 3 private val MaxAncestorsInConversation = 2 override def fsCustomMapInput(query: ForYouQuery): Map[String, Int] = { val userAgeOpt = query.clientContext.userId.map { userId => SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue) } val premium = query.features .map(_.getOrElse(ViewerHasPremiumTier, false)).getOrElse(false) Map( "account_age_in_days" -> userAgeOpt.getOrElse(Int.MaxValue), "premium" -> (if (premium) 1 else 0) ) } override def pipelineRequestTransformer(productPipelineQuery: ForYouQuery): HomeMixerRequest = { HomeMixerRequest( clientContext = productPipelineQuery.clientContext, product = ScoredTweetsProduct, productContext = Some( ScoredTweetsProductContext( productPipelineQuery.deviceContext, productPipelineQuery.seenTweetIds, productPipelineQuery.features.map(_.getOrElse(ServedTweetIdsFeature, Seq.empty)), productPipelineQuery.features.map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty)), productPipelineQuery.features.flatMap(_.getOrElse(SignupCountryFeature, None)), productPipelineQuery.features .flatMap(_.getOrElse(ViewerAllowsForYouRecommendationsFeature, None)), productPipelineQuery.features.flatMap(_.getOrElse(SignupSourceFeature, None)), productPipelineQuery.features.flatMap(_.getOrElse(UserFollowersCountFeature, None)), productPipelineQuery.features .map(_.getOrElse(ServedAuthorIdsFeature, Map.empty[Long, Seq[Long]])) ) ), serializedRequestCursor = productPipelineQuery.pipelineCursor.map(UrtCursorSerializer.serializeCursor), maxResults = None, debugParams = None, homeRequestParam = false ) } override def productPipelineResultTransformer( productPipelineResult: t.ScoredTweetsResponse ): Seq[ScoredTweetWithConversationMetadata] = { val scoredTweets = productPipelineResult.scoredTweets.flatMap { focalTweet => val parentTweets = focalTweet.ancestors.getOrElse(Seq.empty).sortBy(-_.tweetId) val (intermediates, root) = parentTweets.splitAt(parentTweets.size - 1) val truncatedIntermediates = intermediates.take(MaxModuleSize - MaxAncestorsInConversation).reverse val rootScoredTweet: Seq[ScoredTweetWithConversationMetadata] = root.map { ancestor => ScoredTweetWithConversationMetadata( tweetId = ancestor.tweetId, authorId = ancestor.userId, servedType = focalTweet.servedType, conversationId = Some(ancestor.tweetId), conversationFocalTweetId = Some(focalTweet.tweetId), exclusiveConversationAuthorId = focalTweet.exclusiveConversationAuthorId ) } val conversationId = rootScoredTweet.headOption.map(_.tweetId) val tweetsToParents = if (parentTweets.nonEmpty) parentTweets.zip(parentTweets.tail).toMap else Map.empty[ta.TweetAncestor, ta.TweetAncestor] val intermediateScoredTweets = truncatedIntermediates.map { ancestor => ScoredTweetWithConversationMetadata( tweetId = ancestor.tweetId, authorId = ancestor.userId, servedType = focalTweet.servedType, inReplyToTweetId = tweetsToParents.get(ancestor).map(_.tweetId), conversationId = conversationId, conversationFocalTweetId = Some(focalTweet.tweetId), exclusiveConversationAuthorId = focalTweet.exclusiveConversationAuthorId ) } val parentScoredTweets = rootScoredTweet ++ intermediateScoredTweets val conversationFocalTweetId = if (parentScoredTweets.nonEmpty) Some(focalTweet.tweetId) else None val focalScoredTweet = ScoredTweetWithConversationMetadata( tweetId = focalTweet.tweetId, authorId = focalTweet.authorId, score = focalTweet.score, servedType = focalTweet.servedType, sourceTweetId = focalTweet.sourceTweetId, sourceUserId = focalTweet.sourceUserId, quotedTweetId = focalTweet.quotedTweetId, quotedUserId = focalTweet.quotedUserId, inReplyToTweetId = parentScoredTweets.lastOption.map(_.tweetId), inReplyToUserId = focalTweet.inReplyToUserId, directedAtUserId = focalTweet.directedAtUserId, inNetwork = focalTweet.inNetwork, sgsValidLikedByUserIds = focalTweet.sgsValidLikedByUserIds, sgsValidFollowedByUserIds = focalTweet.sgsValidFollowedByUserIds, validLikedByUserIds = focalTweet.validLikedByUserIds, topicId = focalTweet.topicId, topicFunctionalityType = focalTweet.topicFunctionalityType, ancestors = focalTweet.ancestors, conversationId = conversationId, conversationFocalTweetId = conversationFocalTweetId, isReadFromCache = focalTweet.isReadFromCache, exclusiveConversationAuthorId = focalTweet.exclusiveConversationAuthorId, authorIsBlueVerified = focalTweet.authorMetadata.map(_.blueVerified), authorIsGoldVerified = focalTweet.authorMetadata.map(_.goldVerified), authorIsGrayVerified = focalTweet.authorMetadata.map(_.grayVerified), authorIsLegacyVerified = focalTweet.authorMetadata.map(_.legacyVerified), authorIsCreator = focalTweet.authorMetadata.map(_.creator), authorFollowers = focalTweet.authorMetadata.flatMap(_.followers), locationId = focalTweet.locationId, predictionRequestId = focalTweet.predictionRequestId, communityId = focalTweet.communityId, communityName = focalTweet.communityName, listId = focalTweet.listId, listName = focalTweet.listName, isNsfw = focalTweet.isNsfw, visibilityReason = focalTweet.visibilityReason, tweetLanguage = focalTweet.tweetLanguage, tweetText = focalTweet.tweetText, tweetTypeMetrics = focalTweet.tweetTypeMetrics, debugString = focalTweet.debugString, viralContentCreatorFeature = focalTweet.viralContentCreator, hasVideoFeature = focalTweet.hasVideo, videoDurationMsFeature = focalTweet.videoDurationMs, mediaIds = focalTweet.mediaIds, grokAnnotations = focalTweet.grokAnnotations, predictedScores = focalTweet.predictedScores, phoenixPredictedScores = focalTweet.phoenixPredictedScores, sourceSignal = focalTweet.sourceSignal, userActionsSize = focalTweet.userActionsSize, userActionsContainsExplicitSignals = focalTweet.userActionsContainsExplicitSignals, grokContentCreatorFeature = focalTweet.grokContentCreator, gorkContentCreatorFeature = focalTweet.gorkContentCreator, ) parentScoredTweets :+ focalScoredTweet } val dedupedTweets = scoredTweets.groupBy(_.tweetId).map { case (_, duplicateAncestors) => duplicateAncestors.maxBy(_.score.getOrElse(0.0)) } // Sort by tweet id to prevent issues with future assumptions of the root being the first // tweet and the focal being the last tweet in a module. The tweets as a whole do not need // to be sorted overall, only the relative order within modules must be kept. dedupedTweets.toSeq.sortBy(_.tweetId) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredVideoTweetsCategorizedProductCandidateSource.scala ================================================ package com.twitter.home_mixer.product.for_you.candidate_source import com.google.inject.Provider import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.model.request.ScoredVideoTweetsProduct import com.twitter.home_mixer.model.request.ScoredVideoTweetsProductContext import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.core.functional_component.candidate_source.product_pipeline.ProductPipelineCandidateSource import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry import com.twitter.snowflake.id.SnowflakeId import javax.inject.Inject import javax.inject.Singleton @Singleton class ScoredVideoTweetsCategorizedProductCandidateSource @Inject() ( override val productPipelineRegistry: Provider[ProductPipelineRegistry], override val paramsBuilder: Provider[ParamsBuilder]) extends ProductPipelineCandidateSource[ ForYouQuery, HomeMixerRequest, t.ScoredTweetsResponse, ScoredVideoTweetCandidate ] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("ScoredVideoTweetsCategorizedProduct") private val MAX_RESULTS = 20 override def fsCustomMapInput(query: ForYouQuery): Map[String, Int] = { val userAgeOpt = query.clientContext.userId.map { userId => SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue) } userAgeOpt.map("account_age_in_days" -> _).toMap } override def pipelineRequestTransformer(productPipelineQuery: ForYouQuery): HomeMixerRequest = { HomeMixerRequest( clientContext = productPipelineQuery.clientContext, product = ScoredVideoTweetsProduct, productContext = Some( ScoredVideoTweetsProductContext( productPipelineQuery.deviceContext, productPipelineQuery.seenTweetIds, Some(t.VideoType.Categorized), None, None, None )), serializedRequestCursor = None, maxResults = Some(MAX_RESULTS), debugParams = None, homeRequestParam = false ) } override def productPipelineResultTransformer( productPipelineResult: t.ScoredTweetsResponse ): Seq[ScoredVideoTweetCandidate] = { productPipelineResult.scoredTweets.map { scoredTweet => ScoredVideoTweetCandidate( scoredTweet.tweetId, scoredTweet.authorId, scoredTweet.score, scoredTweet.servedType, scoredTweet.aspectRatio ) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredVideoTweetsProductCandidateSource.scala ================================================ package com.twitter.home_mixer.product.for_you.candidate_source import com.google.inject.Provider import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.model.request.ScoredVideoTweetsProduct import com.twitter.home_mixer.model.request.ScoredVideoTweetsProductContext import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.core.functional_component.candidate_source.product_pipeline.ProductPipelineCandidateSource import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry import com.twitter.snowflake.id.SnowflakeId import javax.inject.Inject import javax.inject.Singleton /** * [[ScoredTweetWithConversationMetadata]] **/ case class ScoredVideoTweetCandidate( tweetId: Long, authorId: Long, score: Option[Double], servedType: hmt.ServedType, aspectRatio: Option[Double]) @Singleton class ScoredVideoTweetsProductCandidateSource @Inject() ( override val productPipelineRegistry: Provider[ProductPipelineRegistry], override val paramsBuilder: Provider[ParamsBuilder]) extends ProductPipelineCandidateSource[ ForYouQuery, HomeMixerRequest, t.ScoredTweetsResponse, ScoredVideoTweetCandidate ] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("ScoredVideoTweetsProduct") private val MAX_RESULTS = 20 override def fsCustomMapInput(query: ForYouQuery): Map[String, Int] = { val userAgeOpt = query.clientContext.userId.map { userId => SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue) } userAgeOpt.map("account_age_in_days" -> _).toMap } override def pipelineRequestTransformer(productPipelineQuery: ForYouQuery): HomeMixerRequest = { HomeMixerRequest( clientContext = productPipelineQuery.clientContext, product = ScoredVideoTweetsProduct, productContext = Some( ScoredVideoTweetsProductContext( productPipelineQuery.deviceContext, productPipelineQuery.seenTweetIds, None, None, None, None )), serializedRequestCursor = None, maxResults = Some(MAX_RESULTS), debugParams = None, homeRequestParam = false ) } override def productPipelineResultTransformer( productPipelineResult: t.ScoredTweetsResponse ): Seq[ScoredVideoTweetCandidate] = { productPipelineResult.scoredTweets.map { scoredTweet => ScoredVideoTweetCandidate( scoredTweet.tweetId, scoredTweet.authorId, scoredTweet.score, scoredTweet.servedType, scoredTweet.aspectRatio ) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/StoriesModuleCandidateSource.scala ================================================ package com.twitter.home_mixer.product.for_you.candidate_source import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithModerateTimeout import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyFetcherSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.strato.client.Client import com.twitter.strato.client.Fetcher import com.twitter.strato.generated.client.trendsai.ordered.TrendsModuleOnUserClientColumn import com.twitter.strato.generated.client.trendsai.ordered.TrendsModuleOnUserClientColumn.Key import com.twitter.strato.generated.client.trendsai.ordered.TrendsModuleOnUserClientColumn.Value import com.twitter.strato.graphql.thriftscala.ApiImage import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton case class StoryCandidate( id: Long, title: String, context: String, hook: Option[String], thumbnail: Option[ApiImage], socialProof: Seq[String]) @Singleton class StoriesModuleCandidateSource @Inject() ( @Named(BatchedStratoClientWithModerateTimeout) stratoClient: Client) extends StratoKeyFetcherSource[ TrendsModuleOnUserClientColumn.Key, TrendsModuleOnUserClientColumn.Value, StoryCandidate ] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier(name = "StoriesModule") val fetcher: Fetcher[Key, Unit, Value] = stratoClient.fetcher[ TrendsModuleOnUserClientColumn.Key, TrendsModuleOnUserClientColumn.View, TrendsModuleOnUserClientColumn.Value ](TrendsModuleOnUserClientColumn.Path) override protected def stratoResultTransformer(stratoResult: Value): Seq[StoryCandidate] = stratoResult.items.map { v => StoryCandidate( v.id, v.title, v.context, v.hook, v.thumbnail.map { t => ApiImage(t.url, t.width, t.height) }, v.facepile ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/TuneFeedCandidateSource.scala ================================================ package com.twitter.home_mixer.product.for_you.candidate_source import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.trends.trip.GrokTopicTweetsClientColumn import javax.inject.Inject import javax.inject.Singleton case class TuneFeedRequest( grokTopic: Long, language: Option[String], placeId: Option[Long]) /* * Takes 3 grok topics the user is following at random, and fetches 1 post for each. */ @Singleton class TuneFeedCandidateSource @Inject() ( grokTopicTweetsStratoColumn: GrokTopicTweetsClientColumn) extends CandidateSource[ TuneFeedRequest, TweetCandidate ] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("TuneFeed") private val fetcher = grokTopicTweetsStratoColumn.fetcher private val MaxModuleTweets = 20 override def apply(tuneFeedRequest: TuneFeedRequest): Stitch[Seq[TweetCandidate]] = { val key = GrokTopicTweetsClientColumn.Key( language = tuneFeedRequest.language, placeId = tuneFeedRequest.placeId, topicId = tuneFeedRequest.grokTopic ) fetcher.fetch(key, {}).map { response => response.v .map(_.map { tweetId => TweetCandidate(tweetId) }) .getOrElse(Seq.empty).take(MaxModuleTweets) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/UnifiedTrendsCandidateSource.scala ================================================ package com.twitter.home_mixer.product.for_you.candidate_source import com.twitter.events.recos.{thriftscala => t} import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton case class TrendCandidate(candidate: t.TrendCandidate, rank: Option[Int]) @Singleton class UnifiedTrendsCandidateSource @Inject() ( unifiedCandidatesService: t.EventsRecosService.MethodPerEndpoint) extends CandidateSource[ t.GetUnfiedCandidatesRequest, TrendCandidate ] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("UnifiedTrends") override def apply(request: t.GetUnfiedCandidatesRequest): Stitch[Seq[TrendCandidate]] = { Stitch.callFuture(unifiedCandidatesService.getUnifiedCandidates(request)).map { unifiedCandidateResponse => val trends: Seq[t.TrendCandidate] = unifiedCandidateResponse.candidates.collect { case t.UnifiedCandidate.Trend(trend) => trend } trends.zipWithIndex.map { case (trend, index) => TrendCandidate(trend, Some(index + 1)) } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/ArticlePreviewTextFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.for_you.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.ArticleIdFeature import com.twitter.home_mixer.model.HomeFeatures.ArticlePreviewTextFeature import com.twitter.home_mixer.product.for_you.feature_hydrator.ArticlePreviewTextFeatureHydrator.ArticlePreviewTextColumn import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.strato.client.{Client => StratoClient} import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class ArticlePreviewTextFeatureHydrator @Inject() ( stratoClient: StratoClient) extends CandidateFeatureHydrator[PipelineQuery, BaseTweetCandidate] { private val fetcher = stratoClient.fetcher[Long, Unit, String](ArticlePreviewTextColumn) override val features: Set[Feature[_, _]] = Set(ArticlePreviewTextFeature) override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ArticlePreviewText") override def apply( query: PipelineQuery, candidate: BaseTweetCandidate, existingFeatures: FeatureMap ): Stitch[FeatureMap] = { existingFeatures.get(ArticleIdFeature) match { case Some(articleId) => fetcher.fetch(articleId).map { fetchResult => val articlePreviewText: Option[String] = fetchResult.v FeatureMap(ArticlePreviewTextFeature, articlePreviewText) } case None => Stitch.value(FeatureMap(ArticlePreviewTextFeature, None)) } } } object ArticlePreviewTextFeatureHydrator { final val ArticlePreviewTextColumn: String = "article/fields/previewText.ArticleEntity" } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/AuthorEnabledPreviewsFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.for_you.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorEnabledPreviewsFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.audiencerewards.audienceRewardsService.GetCreatorPreferencesOnUserClientColumn import javax.inject.Inject import javax.inject.Singleton /** * Hydrates the `AuthorEnabledPreviews` feature for tweets authored by creators by querying the * `GetCreatorPreferences` Strato column. This feature corresponds to the `previews_enabled` field of that column. * Given a tweet from a creator, this feature indicates whether that creator has enabled previews * on their profile. */ @Singleton class AuthorEnabledPreviewsFeatureHydrator @Inject() ( getCreatorPreferencesOnUserClientColumn: GetCreatorPreferencesOnUserClientColumn) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("AuthorEnabledPreviews") override val features: Set[Feature[_, _]] = Set(AuthorEnabledPreviewsFeature) private val fetcher = getCreatorPreferencesOnUserClientColumn.fetcher private val DefaultAuthorEnabledPreviewsValue = true override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { val candidateAuthors = candidates .map(_.features.getOrElse(AuthorIdFeature, None)) .toSet .flatten // Build a map of creator -> authorEnabledPreviews, then use it to populate candidate features val authorIdToFeatureStitch = Stitch.collect { candidateAuthors .map { author => val isAuthorEnabledPreviews = fetcher.fetch(author).map { _.v.map(_.previewsEnabled).getOrElse(DefaultAuthorEnabledPreviewsValue) } (author, isAuthorEnabledPreviews) }.toMap } authorIdToFeatureStitch.map { authorIdToFeatureMap => candidates.map { _.features.getOrElse(AuthorIdFeature, None) match { case Some(authorId) => FeatureMapBuilder() .add(AuthorEnabledPreviewsFeature, authorIdToFeatureMap(authorId)) .build() case _ => FeatureMapBuilder() .add(AuthorEnabledPreviewsFeature, DefaultAuthorEnabledPreviewsValue) .build() } } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "audience-rewards/thrift/src/main/thrift/common:thrift-scala", "configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider", "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "recruiting/candidate-service/src/main/thrift:thrift-scala", "servo/repo/src/main/scala", "stitch/stitch-timelineservice", "strato/config/columns/audiencerewards/audienceRewardsService:audienceRewardsService-strato-client", "strato/config/columns/interests:interests-strato-client", "strato/config/columns/recruiting/api/user:user-strato-client", "strato/config/columns/socialgraph/service:exists-strato-client", "strato/config/columns/timelines/pintweet:pintweet-strato-client", "strato/config/columns/tweetypie/federated:federated-strato-client", "strato/config/columns/tweetypie/managed:managed-strato-client", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/CurrentPinnedTweetFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.for_you.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.CurrentPinnedTweetFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.strato.generated.client.timelines.pintweet.PinTweetFannedOutUserCacheClientColumn import javax.inject.Inject import javax.inject.Singleton @Singleton class CurrentPinnedTweetFeatureHydrator @Inject() ( pinTweetFannedOutUserCacheClientColumn: PinTweetFannedOutUserCacheClientColumn) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("CurrentPinnedTweet") override val features: Set[Feature[_, _]] = Set(CurrentPinnedTweetFeature) private val fetcher: Fetcher[ PinTweetFannedOutUserCacheClientColumn.Key, PinTweetFannedOutUserCacheClientColumn.View, PinTweetFannedOutUserCacheClientColumn.Value ] = pinTweetFannedOutUserCacheClientColumn.fetcher override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)).distinct Stitch .collectToTry { authorIds.map { authorId => fetcher.fetch(authorId).map(_.v.map(data => authorId -> data.tweetId)) } }.map { results => val authorPinTweetMap = results.flatMap(_.toOption).flatten.toMap candidates.map { candidate => val authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L) val pinnedTweet = authorPinTweetMap.get(authorId) FeatureMap(CurrentPinnedTweetFeature, pinnedTweet) } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/DisplayedGrokTopicQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.for_you.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.CurrentDisplayedGrokTopicFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.interests.FollowedGrokTopicsWithNameOnUserClientColumn import javax.inject.Inject import javax.inject.Singleton import scala.util.Random @Singleton class DisplayedGrokTopicQueryFeatureHydrator @Inject() ( followedGrokTopicsWithNameOnUserClientColumn: FollowedGrokTopicsWithNameOnUserClientColumn) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "FollowedGrokTopics") private val fetcher = followedGrokTopicsWithNameOnUserClientColumn.fetcher override def features: Set[Feature[_, _]] = Set(CurrentDisplayedGrokTopicFeature) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { fetcher .fetch(query.getRequiredUserId) .map { result => val topicResult = result.v.map(_.toSeq).getOrElse(Seq.empty) val randomTopic = Random.shuffle(topicResult).take(1).headOption FeatureMap(CurrentDisplayedGrokTopicFeature, randomTopic) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FocalTweetFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.for_you.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.FocalTweetRealNamesFeature import com.twitter.home_mixer.model.HomeFeatures.FocalTweetScreenNamesFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton /** * Social context for convo modules is hydrated on the root Tweet but needs info about the focal * Tweet (e.g. author) to render the banner. This hydrator copies focal Tweet data into the root. */ @Singleton class FocalTweetFeatureHydrator @Inject() () extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FocalTweet") override val features: Set[Feature[_, _]] = Set( FocalTweetAuthorIdFeature, FocalTweetInNetworkFeature, FocalTweetRealNamesFeature, FocalTweetScreenNamesFeature ) private val DefaultFeatureMap = FeatureMapBuilder() .add(FocalTweetAuthorIdFeature, None) .add(FocalTweetInNetworkFeature, None) .add(FocalTweetRealNamesFeature, None) .add(FocalTweetScreenNamesFeature, None) .build() override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { // Build a map of all the focal tweets to their corresponding features val focalTweetIdToFeatureMap = candidates.flatMap { candidate => val focalTweetId = candidate.features.getOrElse(ConversationModuleFocalTweetIdFeature, None) if (focalTweetId.contains(candidate.candidate.id)) { Some(candidate.candidate.id -> candidate.features) } else None }.toMap val updatedFeatureMap = candidates.map { candidate => val focalTweetId = candidate.features.getOrElse(ConversationModuleFocalTweetIdFeature, None) val conversationId = candidate.features.getOrElse(ConversationModuleIdFeature, None) // Check if the candidate is a root tweet and ensure its focal tweet's features are available if (conversationId.contains(candidate.candidate.id) && focalTweetId.exists(focalTweetIdToFeatureMap.contains)) { val featureMap = focalTweetIdToFeatureMap.get(focalTweetId.get).get FeatureMapBuilder() .add(FocalTweetAuthorIdFeature, featureMap.getOrElse(AuthorIdFeature, None)) .add(FocalTweetInNetworkFeature, Some(featureMap.getOrElse(InNetworkFeature, true))) .add( FocalTweetRealNamesFeature, Some(featureMap.getOrElse(RealNamesFeature, Map.empty[Long, String]))) .add( FocalTweetScreenNamesFeature, Some(featureMap.getOrElse(ScreenNamesFeature, Map.empty[Long, String]))) .build() } else DefaultFeatureMap } Stitch.value(updatedFeatureMap) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FollowingSportsAccountQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.for_you.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.FollowsSportsAccountFeature import com.twitter.home_mixer.product.for_you.param.ForYouParam.FollowingSportsGateUsersParam import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.socialgraph.{thriftscala => sg} import com.twitter.strato.generated.client.socialgraph.service.ExistsClientColumn import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import javax.inject.Inject import javax.inject.Singleton @Singleton class FollowingSportsAccountsQueryFeatureHydrator @Inject() ( existsClientColumn: ExistsClientColumn) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FollowingSportsAccounts") override val features: Set[Feature[_, _]] = Set(FollowsSportsAccountFeature) val lookupContext = sg.LookupContext(includeInactive = false) private val fetcher: Fetcher[ ExistsClientColumn.Key, ExistsClientColumn.View, ExistsClientColumn.Value ] = existsClientColumn.fetcher override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { query.getOptionalUserId match { case Some(userId) => Stitch .traverse(query.params(FollowingSportsGateUsersParam).toSeq) { sportsAccountId => val request = (userId, Seq(sg.Relationship(sg.RelationshipType.Following)), sportsAccountId) fetcher .fetch(request, lookupContext).map(_.v.getOrElse(false)) }.map { followingAccounts: Seq[Boolean] => FeatureMap(FollowsSportsAccountFeature, followingAccounts.contains(true)) } case None => Stitch.value(FeatureMap(FollowsSportsAccountFeature, false)) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.for_you.feature_hydrator import com.twitter.home_mixer.marshaller.timelines.DeviceContextMarshaller import com.twitter.home_mixer.model.HomeFeatures.TLSOriginalTweetsWithAuthorFeature import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature import com.twitter.home_mixer.model.request.DeviceContext import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.product.for_you.param.ForYouParam.{EnableGetTweetsFromArchiveIndex, EnableTLSHydrationParam} import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.stitch.Stitch import com.twitter.stitch.timelineservice.TimelineService import com.twitter.timelineservice.{thriftscala => t} import javax.inject.Inject import javax.inject.Singleton @Singleton case class TimelineServiceTweetsQueryFeatureHydrator @Inject() ( timelineService: TimelineService, deviceContextMarshaller: DeviceContextMarshaller) extends QueryFeatureHydrator[PipelineQuery with HasDeviceContext] with Conditionally[PipelineQuery with HasDeviceContext] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TimelineServiceTweets") override val features: Set[Feature[_, _]] = Set(TimelineServiceTweetsFeature, TLSOriginalTweetsWithAuthorFeature) override def onlyIf(query: PipelineQuery with HasDeviceContext): Boolean = { query.params(EnableTLSHydrationParam) } private[this] def getTweetsFromArchiveIndexOpt( query: PipelineQuery with HasDeviceContext ): Option[Boolean] = { if (query.params(EnableGetTweetsFromArchiveIndex)) { // Fall back to the default in timelineservice, i.e. get tweets from archive index. None } else { Some(false) } } private val MaxTimelineServiceTweets = 200 override def hydrate(query: PipelineQuery with HasDeviceContext): Stitch[FeatureMap] = { val deviceContext = query.deviceContext.getOrElse(DeviceContext.Empty) val timelineQueryOptions = t.TimelineQueryOptions( contextualUserId = query.clientContext.userId, deviceContext = Some(deviceContextMarshaller(deviceContext, query.clientContext)), getTweetsFromArchiveIndex = getTweetsFromArchiveIndexOpt(query) ) val timelineServiceQuery = t.TimelineQuery( timelineType = t.TimelineType.Home, timelineId = query.getRequiredUserId, maxCount = MaxTimelineServiceTweets.toShort, cursor2 = None, options = Some(timelineQueryOptions), timelineId2 = query.clientContext.userId.map(t.TimelineId(t.TimelineType.Home, _, None)), ) timelineService.getTimeline(timelineServiceQuery).map { timeline => val tweets = timeline.entries.collect { case t.TimelineEntry.Tweet(tweet) => tweet.statusId } // Non replies and non retweets val originalTweetsWithAuthor = timeline.entries.collect { case t.TimelineEntry.Tweet(tweet) if tweet.inReplyToStatusId.isEmpty && tweet.sourceStatusId.isEmpty => (tweet.statusId, tweet.userId) } FeatureMapBuilder() .add(TimelineServiceTweetsFeature, tweets) .add(TLSOriginalTweetsWithAuthorFeature, originalTweetsWithAuthor) .build() } } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(95) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetAuthorFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.for_you.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.{ TLSOriginalTweetsWithAuthorFeature, TLSOriginalTweetsWithConfirmedAuthorFeature } import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.{FeatureMap, FeatureMapBuilder} import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.spam.rtf.{thriftscala => rtf} import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.tweetypie.managed.HomeMixerTesOnTweetClientColumn import com.twitter.strato.client.Client import com.twitter.tweetypie.{thriftscala => tp} import javax.inject.{Inject, Singleton} import com.twitter.util.logging.Logging import javax.inject.Named import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithLongTimeout @Singleton class TweetAuthorFeatureHydrator @Inject() ( @Named(BatchedStratoClientWithLongTimeout) stratoClient: Client, stats: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] with Logging { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetAuthor") override val features: Set[Feature[_, _]] = Set(TLSOriginalTweetsWithConfirmedAuthorFeature) private def fetchTweetDataFromTES(tweetId: Long, tweetFieldsOptions: tp.GetTweetFieldsOptions): Stitch[Option[(Long, Long)]] = { val fetcher = new HomeMixerTesOnTweetClientColumn(stratoClient).fetcher fetcher .fetch(tweetId, tweetFieldsOptions) .map(_.v) .map { case Some(result) => result.tweetResult match { case tp.TweetFieldsResultState.Found(tweetResult) => val isRetweet = tweetResult.retweetedTweet.isDefined tweetResult.tweet.coreData.flatMap { coreData => val isReply = coreData.reply.isDefined if (!isRetweet && !isReply) Some(tweetId -> coreData.userId) else None } case _ => None } case None => None } } override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val originals: Seq[(Long, Option[Long])] = query.features.map(_.getOrElse(TLSOriginalTweetsWithAuthorFeature, Seq())).getOrElse(Seq.empty) if (originals.isEmpty) return Stitch.value( FeatureMapBuilder() .add(TLSOriginalTweetsWithConfirmedAuthorFeature, Seq.empty[(Long, Long)]) .build() ) val missingIds = originals.collect { case (tid, None) => tid } val tweetFieldsOptions = tp.GetTweetFieldsOptions( tweetIncludes = Set(tp.TweetInclude.TweetFieldId(tp.Tweet.CoreDataField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.CountsField.id)), safetyLevel = Some(rtf.SafetyLevel.TimelineHome), forUserId = query.getOptionalUserId ) val fetchedTweetData: Stitch[Map[Long, Long]] = Stitch .collect(missingIds.map { id => fetchTweetDataFromTES(id, tweetFieldsOptions) }) .map(_.flatten.toMap) fetchedTweetData.map { found => val rebuilt: Seq[(Long, Long)] = originals.collect { case (tid, authorOpt) => authorOpt.orElse(found.get(tid)).map(aid => (tid, aid)) }.flatten FeatureMapBuilder() .add(TLSOriginalTweetsWithConfirmedAuthorFeature, rebuilt) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetAuthorFollowersFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.for_you.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.gizmoduck.{thriftscala => gt} import com.twitter.home_mixer.model.HomeFeatures.{ TLSOriginalTweetsWithConfirmedAuthorFeature, TweetAuthorFollowersFeature } import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.{FeatureMap, FeatureMapBuilder} import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import javax.inject.{Inject, Singleton} import com.twitter.util.logging.Logging @Singleton class TweetAuthorFollowersFeatureHydrator @Inject() ( gizmoduck: gt.UserService.MethodPerEndpoint, stats: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] with Logging { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetAuthorFollowers") override val features: Set[Feature[_, _]] = Set(TweetAuthorFollowersFeature) private val queryFields: Set[gt.QueryFields] = Set( gt.QueryFields.Counts ) private val lookupContext = gt.LookupContext(isRequestSheddable = Some(true)) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val originals: Seq[(Long, Long)] = query.features.map(_.getOrElse(TLSOriginalTweetsWithConfirmedAuthorFeature, Seq())).getOrElse(Seq.empty) if (originals.isEmpty) { return Stitch.value( FeatureMapBuilder() .add(TweetAuthorFollowersFeature, Map.empty[Long, Option[Long]]) .build() ) } val authorIds = originals.map(_._2).distinct OffloadFuturePools.offloadFuture { gizmoduck.get(lookupContext, authorIds, queryFields).map { gizmoduckResponse => val hydratedUsersMap = gizmoduckResponse.collect { case userResult if userResult.user.isDefined => val user = userResult.user.get user.id -> user }.toMap.mapValues(user => user.counts.map(_.followers)) val tweetAuthorFollowersMap = originals.map { case (tweetId, authorId) => tweetId -> hydratedUsersMap.getOrElse(authorId, None) }.toMap FeatureMapBuilder() .add(TweetAuthorFollowersFeature, tweetAuthorFollowersMap) .build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetEngagementsFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.for_you.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.{FeatureMap, FeatureMapBuilder} import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.tweetypie.federated.ApiCountsOnTweetClientColumn import com.twitter.strato.client.Client import javax.inject.{Inject, Singleton} import com.twitter.util.logging.Logging import javax.inject.Named import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithLongTimeout case class TweetEngagementCounts( favoriteCount: Option[Long], replyCount: Option[Long], retweetCount: Option[Long], quoteCount: Option[Long], bookmarkCount: Option[Long] ) object TweetEngagementCountsFeature extends Feature[PipelineQuery, Map[Long, TweetEngagementCounts]] @Singleton class TweetEngagementsFeatureHydrator @Inject() ( @Named(BatchedStratoClientWithLongTimeout) stratoClient: Client, stats: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] with Logging { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetEngagements") override val features: Set[Feature[_, _]] = Set(TweetEngagementCountsFeature) private val engagementCountsFoundCounter = stats.counter("TweetEngagementsFound") private val engagementCountsNotFoundCounter = stats.counter("TweetEngagementsNotFound") private def fetchEngagementCountsFromStrato(tweetId: Long): Stitch[Option[TweetEngagementCounts]] = { val fetcher = new ApiCountsOnTweetClientColumn(stratoClient).fetcher fetcher .fetch(tweetId) .map(_.v) .map { case Some(result) => engagementCountsFoundCounter.incr() Some(TweetEngagementCounts( favoriteCount = result.favoriteCount, replyCount = result.replyCount, retweetCount = result.retweetCount, quoteCount = result.quoteCount, bookmarkCount = result.bookmarkCount )) case None => engagementCountsNotFoundCounter.incr() None } } override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val tweetIds: Seq[Long] = query.features.map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty[Long])).getOrElse(Seq.empty) if (tweetIds.isEmpty) return Stitch.value( FeatureMapBuilder() .add(TweetEngagementCountsFeature, Map.empty[Long, TweetEngagementCounts]) .build() ) val fetchedEngagementCounts: Stitch[Map[Long, TweetEngagementCounts]] = Stitch .collect(tweetIds.map { id => fetchEngagementCountsFromStrato(id).map(_.map(counts => id -> counts)) }) .map(_.flatten.toMap) fetchedEngagementCounts.map { found => FeatureMapBuilder() .add(TweetEngagementCountsFeature, found) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.for_you.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.ArticleIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableTweetEntityServiceMigrationParam import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithLongTimeout import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.spam.rtf.{thriftscala => rtf} import com.twitter.stitch.Stitch import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} import com.twitter.strato.client.Client import com.twitter.strato.generated.client.tweetypie.managed.HomeMixerOnTweetClientColumn import com.twitter.tweetypie.{thriftscala => TP} import javax.inject.Named import javax.inject.Inject import javax.inject.Singleton import com.twitter.finagle.stats.StatsReceiver @Singleton class TweetPreviewTweetypieCandidateFeatureHydrator @Inject() ( statsReceiver: StatsReceiver, tweetypieStitchClient: TweetypieStitchClient, @Named(BatchedStratoClientWithLongTimeout) stratoClient: Client) extends CandidateFeatureHydrator[PipelineQuery, BaseTweetCandidate] { private val tweetypieTweetsFoundCounter = statsReceiver.counter("TweetPreviewTweetypieTweetsFound") private val tweetypieTweetsNotFoundCounter = statsReceiver.counter("TweetPreviewTweetypieTweetsNotFound") private val tesTweetsFoundCounter = statsReceiver.counter("TweetPreviewTesTweetsFound") private val tesTweetsNotFoundCounter = statsReceiver.counter("TweetPreviewTesTweetsNotFound") private val CoreTweetFields: Set[TP.TweetInclude] = Set[TP.TweetInclude]( TP.TweetInclude.TweetFieldId(TP.Tweet.IdField.id), TP.TweetInclude.TweetFieldId(TP.Tweet.CoreDataField.id), TP.TweetInclude.TweetFieldId(TP.Tweet.ArticleField.id), ) private val DefaultFeatureMap = FeatureMapBuilder() .add(TweetTextFeature, None) .add(IsHydratedFeature, false) .add(AuthorIdFeature, None) .add(VisibilityReason, None) .add(ArticleIdFeature, None) .build() override val features: Set[Feature[_, _]] = Set(TweetTextFeature, IsHydratedFeature, AuthorIdFeature, VisibilityReason, ArticleIdFeature) override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetPreviewTweetypie") private def buildFeatureMap( gtfResult: Stitch[TP.GetTweetFieldsResult], isTes: Boolean ): Stitch[FeatureMap] = { gtfResult.map { case TP.GetTweetFieldsResult(_, TP.TweetFieldsResultState.Found(found), _, _) => if (isTes) tesTweetsFoundCounter.incr() else tweetypieTweetsFoundCounter.incr() val tweetText = found.tweet.coreData.map(_.text) FeatureMapBuilder(sizeHint = 5) .add(TweetTextFeature, tweetText) .add(IsHydratedFeature, true) .add(AuthorIdFeature, found.tweet.coreData.map(_.userId)) .add(VisibilityReason, found.suppressReason) .add(ArticleIdFeature, found.tweet.article.map(_.id)) .build() case _ => if (isTes) tesTweetsNotFoundCounter.incr() else tweetypieTweetsNotFoundCounter.incr() DefaultFeatureMap } } override def apply( query: PipelineQuery, candidate: BaseTweetCandidate, existingFeatures: FeatureMap ): Stitch[FeatureMap] = { val getTweetFieldsOptions = TP.GetTweetFieldsOptions( tweetIncludes = CoreTweetFields, includeRetweetedTweet = false, includeQuotedTweet = false, visibilityPolicy = TP.TweetVisibilityPolicy.UserVisible, safetyLevel = Some(rtf.SafetyLevel.TimelineHomeTweetPreview), forUserId = query.getOptionalUserId ) if (query.params(EnableTweetEntityServiceMigrationParam)) { val fetcher = new HomeMixerOnTweetClientColumn(stratoClient).fetcher fetcher .fetch( candidate.id, getTweetFieldsOptions ).map(_.v).flatMap { case Some(result) => buildFeatureMap(Stitch.value(result), true) case None => tesTweetsNotFoundCounter.incr() Stitch.value(DefaultFeatureMap) } } else { val gtfResult: Stitch[TP.GetTweetFieldsResult] = tweetypieStitchClient .getTweetFields( tweetId = candidate.id, options = getTweetFieldsOptions ) buildFeatureMap(gtfResult, false) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/ViewerHasJobRecommendationsFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.for_you.feature_hydrator import com.twitter.candidateservice.core_io.MatchingProfile import com.twitter.home_mixer.model.HomeFeatures.ViewerHasJobRecommendationsEnabled import com.twitter.home_mixer.model.HomeFeatures.ViewerHasRecruitingOrganizationRecommendationsEnabled import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithDefaultTimeout import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableViewerHasJobRecommendationsFeatureParam import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoErrCategorizer import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.client.Client import com.twitter.strato.generated.client.recruiting.api.user.MatchingProfileOnUserClientColumn import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class ViewerHasJobRecommendationsFeatureHydrator @Inject() ( @Named(BatchedStratoClientWithDefaultTimeout) stratoClient: Client) extends QueryFeatureHydrator[PipelineQuery] with Conditionally[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ViewerHasJobRecommendations") override val features: Set[Feature[_, _]] = Set(ViewerHasJobRecommendationsEnabled, ViewerHasRecruitingOrganizationRecommendationsEnabled) override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableViewerHasJobRecommendationsFeatureParam) private val fetcher = stratoClient.fetcher[ Long, Unit, MatchingProfile ](MatchingProfileOnUserClientColumn.Path) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId fetcher .fetch(userId, ()) .map { result => val (orgsEnabled, jobsEnabled) = result.v match { case None => // When no matching profile exists, the user cannot have consented // to see job recommendations. Just show org recommendations. (true, false) case Some(profile) => // When a profile does exist, check for consent. Consent cannot be withdrawn. // It indicates that at some point in the past, the user has opted in to see // job recommendations. They can later toggle off recommendations. Possible // states: // // 1. User consented and recommendations enabled // (a) there are recommendations --> show job recommendations // (b) there are no recommendations --> show org recommendations // 2. User toggled off recommendations --> show nothing // 3. User has not consented --> show org recommendations if (!profile.recommendationsEnabled) { (false, false) } else { val showJobRecommendations = profile.consentedAt.nonEmpty && profile.hasJobRecommendations (!showJobRecommendations, showJobRecommendations) } } FeatureMapBuilder() .add(ViewerHasRecruitingOrganizationRecommendationsEnabled, orgsEnabled) .add(ViewerHasJobRecommendationsEnabled, jobsEnabled) .build() } .rescue(StratoErrCategorizer.CategorizeStratoException) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/trends_events", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/NotArticleFilter.scala ================================================ package com.twitter.home_mixer.product.for_you.filter import com.twitter.home_mixer.model.HomeFeatures.IsArticleFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Filter out tweets that contain articles */ object NotArticleFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("NotArticle") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (kept, removed) = candidates.partition { candidate => !candidate.features.getOrElse(IsArticleFeature, false) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/PromotedTrendFilter.scala ================================================ package com.twitter.home_mixer.product.for_you.filter import com.twitter.product_mixer.component_library.model.candidate.trends_events.PromotedTrendNameFeature import com.twitter.product_mixer.component_library.model.candidate.trends_events.UnifiedTrendCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * A filter that drops promoted trends */ object PromotedTrendFilter extends Filter[PipelineQuery, UnifiedTrendCandidate] { override val identifier = FilterIdentifier("PromotedTrend") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[UnifiedTrendCandidate]] ): Stitch[FilterResult[UnifiedTrendCandidate]] = { val filterResult = { val (removed, kept) = candidates.partition { candidate => candidate.features.get(PromotedTrendNameFeature).isDefined } FilterResult(kept.map(_.candidate), removed.map(_.candidate)) } Stitch.value(filterResult) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/SocialContextFilter.scala ================================================ package com.twitter.home_mixer.product.for_you.filter import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.product.for_you.param.ForYouParam import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelineservice.suggests.{thriftscala => st} object SocialContextFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("SocialContext") // Tweets from candidate sources which don't need generic like/follow/topic proof private val AllowedSources: Set[st.SuggestType] = Set( st.SuggestType.RankedListTweet, st.SuggestType.RecommendedTrendTweet, st.SuggestType.MediaTweet ) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val enableIsVerifiedAuthorFilter = query.params(ForYouParam.EnableVerifiedAuthorSocialContextBypassParam) val enableTopicSocialContextFilter = query.params(ForYouParam.EnableTopicSocialContextFilterParam) val validTweetIds = candidates .filter { candidate => candidate.features.getOrElse(InNetworkFeature, true) || candidate.features.getOrElse(SuggestTypeFeature, None).exists(AllowedSources.contains) || candidate.features.getOrElse(ConversationModuleFocalTweetIdFeature, None).isDefined || (enableIsVerifiedAuthorFilter && isVerifiedAuthor(candidate.features)) || hasLikedBySocialContext(candidate.features) || hasFollowedBySocialContext(candidate.features) || (enableTopicSocialContextFilter && hasTopicSocialContext(candidate.features)) }.map(_.candidate.id).toSet val (kept, removed) = candidates.map(_.candidate).partition(candidate => validTweetIds.contains(candidate.id)) Stitch.value(FilterResult(kept = kept, removed = removed)) } private def isVerifiedAuthor(candidateFeatures: FeatureMap): Boolean = { candidateFeatures.getOrElse(AuthorIsBlueVerifiedFeature, false) || candidateFeatures.getOrElse(AuthorIsGoldVerifiedFeature, false) || candidateFeatures.getOrElse(AuthorIsGrayVerifiedFeature, false) || candidateFeatures.getOrElse(AuthorIsLegacyVerifiedFeature, false) } private def hasLikedBySocialContext(candidateFeatures: FeatureMap): Boolean = candidateFeatures .getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) .exists( candidateFeatures .getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty) .toSet.contains ) private def hasFollowedBySocialContext(candidateFeatures: FeatureMap): Boolean = candidateFeatures.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty private def hasTopicSocialContext(candidateFeatures: FeatureMap): Boolean = { candidateFeatures.getOrElse(TopicIdSocialContextFeature, None).isDefined && candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.scala ================================================ package com.twitter.home_mixer.product.for_you.filter import com.twitter.home_mixer.model.HomeFeatures.ArticlePreviewTextFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object TweetPreviewTextFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("TweetPreviewText") private val PreviewTextLength = 50 private val MinTweetLength = PreviewTextLength * 2 private val MaxNewlines = 2 private val HttpPrefix = "http://" private val HttpsPrefix = "https://" override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (kept, removed) = candidates .partition { candidate => val text = candidate.features .get(TweetTextFeature).orElse( candidate.features.getOrElse(ArticlePreviewTextFeature, None)).getOrElse("") text.length > MinTweetLength && text.take(PreviewTextLength).count(_ == '\n') <= MaxNewlines && !(text.startsWith(HttpPrefix) || text.startsWith(HttpsPrefix)) } val filterResult = FilterResult( kept = kept.map(_.candidate), removed = removed.map(_.candidate) ) Stitch.value(filterResult) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/PushToHomeRequestGate.scala ================================================ package com.twitter.home_mixer.product.for_you.functional_component.gate import com.twitter.home_mixer.product.for_you.model.ForYouQuery import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.stitch.Stitch /** * Continues when the request is a Push-To-Home notification request */ object PushToHomeRequestGate extends Gate[ForYouQuery] { override val identifier: GateIdentifier = GateIdentifier("PushToHomeRequest") override def shouldContinue(query: ForYouQuery): Stitch[Boolean] = Stitch.value(query.pushToHomeTweetId.isDefined) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", ], exports = [ "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/FollowingSportsUsersGate.scala ================================================ package com.twitter.home_mixer.product.for_you.gate import com.twitter.home_mixer.model.HomeFeatures.FollowsSportsAccountFeature import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Continue for all users that follow a sports account */ object FollowingSportsUserGate extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("FollowingSportsUser") override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { val followsSportsAccount = query.features.map(_.getOrElse(FollowsSportsAccountFeature, false)) Stitch.value(followsSportsAccount.contains(true)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/TuneFeedModuleGate.scala ================================================ package com.twitter.home_mixer.product.for_you.gate import com.twitter.home_mixer.model.HomeFeatures.CurrentDisplayedGrokTopicFeature import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object TuneFeedModuleGate extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("TuneFeed") override def shouldContinue( query: PipelineQuery ): Stitch[Boolean] = { val grokTopicToDisplay = query.features.get.getOrElse(CurrentDisplayedGrokTopicFeature, None) Stitch.value(grokTopicToDisplay.isDefined) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/gate/UserFollowingRangeGate.scala ================================================ package com.twitter.home_mixer.product.for_you.gate import com.twitter.home_mixer.model.HomeFeatures.UserFollowingCountFeature import com.twitter.home_mixer.product.for_you.param.ForYouParam.{MinFollowingCountParam, MaxFollowingCountParam} import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Gate that checks if the user's following count is within a valid range. * The range is controlled by configurable parameters for lower and upper bounds. */ object UserFollowingRangeGate extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("UserFollowingRange") override def shouldContinue( query: PipelineQuery ): Stitch[Boolean] = { val userFollowingCount = query.features.get.getOrElse(UserFollowingCountFeature, None) val lowerBound = query.params(MinFollowingCountParam) val upperBound = query.params(MaxFollowingCountParam) val isInRange = userFollowingCount match { case Some(count) => count >= lowerBound && count <= upperBound case None => false // If we don't have the following count, don't continue } Stitch.value(isInRange) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", ], exports = [ "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.scala ================================================ package com.twitter.home_mixer.product.for_you.model import com.twitter.adserver.thriftscala.HomeTimelineType import com.twitter.adserver.thriftscala.TimelineRequestParams import com.twitter.dspbidder.commons.{thriftscala => dsp} import com.twitter.home_mixer.model.HomeAdsQuery import com.twitter.home_mixer.model.request.DeviceContext import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.model.request.HasSeenTweetIds import com.twitter.onboarding.task.service.{thriftscala => ots} import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.transformer.HasFlipInjectionParams import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.request._ import com.twitter.product_mixer.core.pipeline.HasPipelineCursor import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.configapi.Params case class ForYouQuery( override val params: Params, override val clientContext: ClientContext, override val pipelineCursor: Option[UrtOrderedCursor], override val requestedMaxResults: Option[Int], override val debugOptions: Option[DebugOptions], override val features: Option[FeatureMap], override val deviceContext: Option[DeviceContext], override val seenTweetIds: Option[Seq[Long]], override val dspClientContext: Option[dsp.DspClientContext]) extends PipelineQuery with HasPipelineCursor[UrtOrderedCursor] with HasDeviceContext with HasSeenTweetIds with HasFlipInjectionParams with HomeAdsQuery { override val product: Product = ForYouProduct override def withFeatureMap(features: FeatureMap): ForYouQuery = copy(features = Some(features)) override val timelineRequestParams: Option[TimelineRequestParams] = Some(TimelineRequestParams(homeTimelineType = Some(HomeTimelineType.Home))) // Fields below are used for FLIP Injection in Onboarding Task Service (OTS) override val displayLocation: ots.DisplayLocation = ots.DisplayLocation.HomeTimeline override val rankingDisablerWithLatestControlsAvailable: Option[Boolean] = None override val isEmptyState: Option[Boolean] = None override val isFirstRequestAfterSignup: Option[Boolean] = None override val isEndOfTimeline: Option[Boolean] = None override val timelineId: Option[Long] = None } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouTweetsResponse.scala ================================================ package com.twitter.home_mixer.product.for_you.model import com.twitter.product_mixer.core.model.marshalling.HasMarshalling case class ForYouTweetsResponse(tweetCandidates: Seq[Long]) extends HasMarshalling ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/communities_to_join", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.scala ================================================ package com.twitter.home_mixer.product.for_you.param import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.param.decider.DeciderKey import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.WhoToFollowModuleDisplayType import com.twitter.product_mixer.component_library.pipeline.candidate.communities_to_join.CommunityToJoinModuleDisplayType import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowUserDisplayType import com.twitter.product_mixer.core.functional_component.configapi.StaticParam import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSEnumParam import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.timelines.configapi.decider.BooleanDeciderParam import com.twitter.util.Duration object ForYouParam { val SupportedClientFSName = "for_you_supported_client" val StaticParamValueZero = StaticParam(0) val StaticParamValueFive = StaticParam(5) object EnableWhoToFollowCandidatePipelineParam extends FSParam[Boolean]( name = "for_you_enable_who_to_follow", default = true ) object EnableWhoToSubscribeCandidatePipelineParam extends FSParam[Boolean]( name = "for_you_enable_who_to_subscribe", default = false ) object EnableTweetPreviewsCandidatePipelineParam extends FSParam[Boolean]( name = "for_you_enable_tweet_previews_candidate_pipeline", default = false ) object ServerMaxResultsParam extends FSBoundedParam[Int]( name = "for_you_server_max_results", default = 35, min = 1, max = 500 ) object AdsNumOrganicItemsParam extends FSBoundedParam[Int]( name = "for_you_ads_num_organic_items", default = 35, min = 1, max = 100 ) object WhoToFollowMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_who_to_follow_min_injection_interval_in_minutes", default = 1800.minutes, min = 0.minutes, max = 6000.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object WhoToFollowDisplayTypeIdParam extends FSEnumParam[WhoToFollowModuleDisplayType.type]( name = "for_you_enable_who_to_follow_display_type_id", default = WhoToFollowModuleDisplayType.Vertical, enum = WhoToFollowModuleDisplayType ) object WhoToFollowUserDisplayTypeIdParam extends FSEnumParam[WhoToFollowUserDisplayType.type]( name = "for_you_enable_who_to_follow_user_display_type_id", default = WhoToFollowUserDisplayType.User, enum = WhoToFollowUserDisplayType ) object WhoToFollowDisplayLocationParam extends FSParam[String]( name = "for_you_who_to_follow_display_location", default = "timeline" ) object WhoToSubscribeMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_who_to_subscribe_min_injection_interval_in_minutes", default = 1800.minutes, min = 0.minutes, max = 6000.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object WhoToSubscribeDisplayTypeIdParam extends FSEnumParam[WhoToFollowModuleDisplayType.type]( name = "for_you_enable_who_to_subscribe_display_type_id", default = WhoToFollowModuleDisplayType.Vertical, enum = WhoToFollowModuleDisplayType ) object TweetPreviewsMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_tweet_previews_min_injection_interval_in_minutes", default = 2.hours, min = 0.minutes, max = 600.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object TweetPreviewsMaxCandidatesParam extends FSBoundedParam[Int]( name = "for_you_tweet_previews_max_candidates", default = 1, min = 0, max = 1 ) object EnableFlipInjectionModuleCandidatePipelineParam extends FSParam[Boolean]( name = "for_you_enable_flip_inline_injection_module", default = true ) object ClearCache { object PtrEnableParam extends FSParam[Boolean]( name = "for_you_clear_cache_ptr_enable", default = false ) object ColdStartEnableParam extends FSParam[Boolean]( name = "for_you_clear_cache_cold_start_enable", default = false ) object WarmStartEnableParam extends FSParam[Boolean]( name = "for_you_clear_cache_warm_start_enable", default = false ) object ManualRefreshEnableParam extends FSParam[Boolean]( name = "for_you_clear_cache_manual_refresh_enable", default = false ) object NavigateEnableParam extends FSParam[Boolean]( name = "for_you_clear_cache_navigate_enable", default = false ) object ColdStartRetainViewportParam extends FSParam[Boolean]( name = "for_you_clear_cache_retain_viewport", default = false ) case object MinEntriesParam extends FSBoundedParam[Int]( name = "for_you_clear_cache_min_entries", default = 10, min = 0, max = 35 ) } object Navigation { object PtrEnableParam extends FSParam[Boolean]( name = "for_you_navigation_ptr_enable", default = false ) object ColdStartEnableParam extends FSParam[Boolean]( name = "for_you_navigation_cold_start_enable", default = false ) object WarmStartEnableParam extends FSParam[Boolean]( name = "for_you_navigation_warm_start_enable", default = false ) object ManualRefreshEnableParam extends FSParam[Boolean]( name = "for_you_navigation_manual_refresh_enable", default = false ) object NavigateEnableParam extends FSParam[Boolean]( name = "for_you_navigation_navigate_enable", default = false ) } /** * This author ID list is used purely for realtime metrics collection around how often we * are serving posts from these authors and which sources they are coming from. */ object AuthorListForStatsParam extends FSParam[Set[Long]]( name = "for_you_author_list_for_stats", default = Set.empty ) object ExperimentStatsParam extends FSParam[String]( name = "for_you_experiment_stats", default = "" ) object CommunitiesToJoinMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_communities_to_join_min_injection_interval_in_minutes", default = 2100.minutes, min = 0.minutes, max = 6000.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object MaxCommunitiesToJoinCandidatesParam extends FSBoundedParam[Int]( name = "for_you_communities_to_join_max_candidates", default = 3, min = 1, max = 10) object CommunitiesToJoinDisplayTypeIdParam extends FSEnumParam[CommunityToJoinModuleDisplayType.type]( name = "for_you_communities_to_join_display_type_id", default = CommunityToJoinModuleDisplayType.Carousel, enum = CommunityToJoinModuleDisplayType ) object EnableCommunitiesToJoinCandidatePipelineParam extends FSParam[Boolean]( name = "for_you_enable_communities_to_join", default = false ) object RecommendedJobMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_recommended_job_min_injection_interval_in_minutes", default = 2100.minutes, min = 0.minutes, max = 6000.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object MaxRecommendedJobCandidatesParam extends FSBoundedParam[Int]( name = "for_you_recommended_job_max_candidates", default = 3, min = 1, max = 10) object EnableRecommendedJobsParam extends FSParam[Boolean]( name = "for_you_enable_recommended_jobs", default = false ) object EnableRecommendedRecruitingOrganizationsParam extends FSParam[Boolean]( name = "for_you_enable_recommended_recruiting_organizations", default = false ) object RecommendedRecruitingOrganizationMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_recommended_recruiting_organization_min_injection_interval_in_minutes", default = 2100.minutes, min = 0.minutes, max = 6000.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object MaxRecommendedRecruitingOrganizationCandidatesParam extends FSBoundedParam[Int]( name = "for_you_recommended_recruiting_organization_max_candidates", default = 3, min = 1, max = 10 ) object EnableBookmarksCandidatePipelineParam extends FSParam[Boolean]( name = "for_you_enable_bookmarks_module", default = false ) object EnablePinnedTweetsCandidatePipelineParam extends FSParam[Boolean]( name = "for_you_enable_pinned_tweets_pipeline", default = false ) object EnableEntryPointPivotParam extends FSParam[Boolean]( name = "for_you_enable_entry_point_pivot", default = false ) object EnableGrokEntryPointPivotParam extends FSParam[Boolean]( name = "for_you_enable_grok_entry_point_pivot", default = false ) object PinnedTweetsModuleMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_pinned_tweets_module_min_injection_interval_in_minutes", default = 1440.minutes, // 1 day min = 0.minutes, max = 6000.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object EnableExplorationTweetsCandidatePipelineParam extends FSParam[Boolean]( name = "for_you_enable_exploration_tweets_pipeline", default = false ) object EnableJetfuelFramePipelineParam extends FSParam[Boolean]( name = "for_you_enable_jetfuel_frame_pipeline", default = false ) object FollowingSportsGateUsersParam extends FSParam[Set[Long]]( name = "for_you_following_sports_users_list", default = Set.empty ) object ExplorationTweetsTimelinePosition extends FSBoundedParam[Int]( name = "for_you_exploration_tweets_position", default = 15, min = 0, max = 50 ) object SuperbowlModuleTimelinePosition extends FSBoundedParam[Int]( name = "for_you_superbowl_position", default = 3, min = 0, max = 50 ) object GrokPivotModuleTimelinePosition extends FSBoundedParam[Int]( name = "for_you_grok_pivot_position", default = 3, min = 0, max = 50 ) object EntryPointPivotMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_entry_point_pivot_min_injection_interval_in_minutes", default = 30.minutes, min = 0.minutes, max = 6000.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object GrokEntryPointPivotMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_grok_entry_point_pivot_min_injection_interval_in_minutes", default = 30.minutes, min = 0.minutes, max = 6000.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object MaxNumberExplorationTweetsParam extends FSBoundedParam[Int]( name = "for_you_exploration_tweets_max_number", default = 2, min = 1, max = 10 ) object EnableBookmarksModuleWeekendGate extends FSParam[Boolean]( name = "for_you_enable_bookmarks_module_weekend_gate", default = false ) object BookmarksModuleMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_bookmarks_module_min_injection_interval_in_minutes", default = 2880.minutes, // 2 days min = 0.minutes, max = 6000.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object InNetworkExplorationTweetsMinInjectionIntervalParam extends FSBoundedParam[Duration]( name = "for_you_in_network_exploration_tweets_min_injection_interval_in_minutes", default = 30.minutes, min = 0.minutes, max = 60.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object ExplorationTweetsMaxFollowerCountParam extends FSBoundedParam[Long]( name = "for_you_exploration_tweets_max_follower_count", default = Long.MaxValue, min = 0L, max = Long.MaxValue ) object EnableViewerHasJobRecommendationsFeatureParam extends FSParam[Boolean]( name = "for_you_enable_viewer_has_job_recommendations_feature", default = false ) object EnableTrendsParam extends FSParam[Boolean]( name = "for_you_enable_trends", default = false ) object EnableKeywordTrendsParam extends FSParam[Boolean]( name = "for_you_enable_keyword_trends", default = false ) object MaxNumberKeywordTrendsParam extends FSBoundedParam[Int]( name = "for_you_keyword_trends_max_number", default = 5, min = 1, max = 10 ) object TrendsModuleMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_trends_min_injection_interval_in_minutes", default = 360.minutes, min = 0.minutes, max = 2880.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object KeywordTrendsModuleMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_keyword_trends_min_injection_interval_in_minutes", default = 360.minutes, min = 0.minutes, max = 2880.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object EnableScoredVideoTweetsCandidatePipelineParam extends FSParam[Boolean]( name = "for_you_enable_scored_video_tweets_pipeline", default = false ) object VideoTweetsModuleMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_video_tweets_module_min_injection_interval_in_minutes", default = 1.hour, min = 0.minutes, max = 6000.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object VideoCarouselNumTweetCandidatesToDedupeAgainstParam extends FSBoundedParam[Int]( name = "for_you_video_carousel_num_tweet_candidates_to_dedupe_against", default = 10, min = 0, max = 50 ) object VideoCarouselTimelinePosition extends FSBoundedParam[Int]( name = "for_you_video_carousel_position", default = 5, min = 0, max = 50 ) object VideoCarouselNumCandidates extends FSBoundedParam[Int]( name = "for_you_video_carousel_num_candidates", default = 5, min = 0, max = 50 ) object VideoCarouselEnableFooterParam extends FSParam[Boolean]( name = "for_you_video_carousel_enable_footer", default = false ) object VideoCarouselAllowVerticalVideos extends FSParam[Boolean]( name = "for_you_video_carousel_allow_vertical_videos", default = true ) object VideoCarouselAllowHorizontalVideos extends FSParam[Boolean]( name = "for_you_video_carousel_allow_horizontal_videos", default = true ) object EnableTLSHydrationParam extends FSParam[Boolean]( name = "for_you_enable_tls_hydration", default = false ) object EnableGetTweetsFromArchiveIndex extends BooleanDeciderParam(decider = DeciderKey.EnableGetTweetsFromArchiveIndex) // Currently only supported by rweb object EnableArticlePreviewTextHydrationParam extends FSParam[Boolean]( name = "for_you_enable_article_preview_hydration", default = false ) object EnableAdsDebugParam extends FSParam[Boolean]( name = "for_you_enable_ads_debug", default = false ) object RelevancePromptEnableParam extends FSParam[Boolean]( name = "for_you_relevance_prompt_enable", default = false ) object RelevancePromptMinInjectionIntervalParam extends FSBoundedParam[Duration]( name = "for_you_relevance_prompt_min_injection_interval_minutes", default = 1440.minutes, min = 0.minutes, max = 144000.minutes ) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object RelevancePromptTweetPositionParam extends FSBoundedParam[Int]( name = "for_you_relevance_prompt_position", default = 15, min = 0, max = 10000 ) object RelevancePromptTitleParam extends FSParam[String]( name = "for_you_relevance_prompt_title", default = "" ) object RelevancePromptPositiveParam extends FSParam[String]( name = "for_you_relevance_prompt_positive", default = "" ) object RelevancePromptNegativeParam extends FSParam[String]( name = "for_you_relevance_prompt_negative", default = "" ) object RelevancePromptNeutralParam extends FSParam[String]( name = "for_you_relevance_prompt_neutral", default = "" ) object EnableForYouTopicSelectorParam extends FSParam[Boolean]( name = "for_you_topic_selector_enabled", default = false ) object ForYouTopicSelectorJetfuelRouteParam extends FSParam[String]( name = "for_you_topic_selector_jetfuel_route", default = "" ) object ForYouTopicSelectorPosition extends FSBoundedParam[Int]( name = "for_you_topic_selector_position", default = 1, min = 0, max = 1000 ) object ForYouAppUpsellJetfuelRouteParam extends FSParam[String]( name = "for_you_app_upsell_jetfuel_route", default = "/cards/pivots/appInstallPivot" ) object EnableForYouAppUpsellParam extends FSParam[Boolean]( name = "for_you_enable_app_upsell", default = false ) object ForYouAppUpsellPosition extends FSBoundedParam[Int]( name = "for_you_app_upsell_position", default = 1, min = 0, max = 1000 ) object EnableTuneFeedCandidatePipelineParam extends FSParam[Boolean]( name = "for_you_enable_tune_feed_pipeline", default = false ) object TuneFeedTimelinePosition extends FSBoundedParam[Int]( name = "for_you_tune_feed_position", default = 7, min = 0, max = 50 ) object TuneFeedModuleMinInjectionIntervalParam extends FSBoundedParam[Duration]( "for_you_tune_feed_module_min_injection_interval_in_minutes", default = 180.minutes, // 3 hours min = 0.minutes, max = 6000.minutes) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object EnableFollowedGrokTopicsHydrationParam extends FSParam[Boolean]( name = "for_you_enable_followed_grok_topics_hydration", default = false ) object EnableForYouTimelineAdsSurface extends FSParam[Boolean]( name = "for_you_enable_timeline_ads_surface", default = false ) object MinFollowingCountParam extends FSBoundedParam[Int]( name = "for_you_user_following_range_gate_min_following_count", default = 0, min = 0, max = 1000000 ) object MaxFollowingCountParam extends FSBoundedParam[Int]( name = "for_you_user_following_range_gate_max_following_count", default = 10000, min = 0, max = 1000000 ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.scala ================================================ package com.twitter.home_mixer.product.for_you.param import com.twitter.home_mixer.param.decider.DeciderKey import com.twitter.home_mixer.product.for_you.param.ForYouParam._ import com.twitter.product_mixer.core.product.ProductParamConfig import com.twitter.servo.decider.DeciderKeyName import javax.inject.Inject import javax.inject.Singleton @Singleton class ForYouParamConfig @Inject() () extends ProductParamConfig { override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableForYouProduct override val supportedClientFSName: String = SupportedClientFSName override val booleanFSOverrides = Seq( ClearCache.PtrEnableParam, ClearCache.ColdStartEnableParam, ClearCache.WarmStartEnableParam, ClearCache.ManualRefreshEnableParam, ClearCache.NavigateEnableParam, ClearCache.ColdStartRetainViewportParam, EnableCommunitiesToJoinCandidatePipelineParam, EnableBookmarksCandidatePipelineParam, EnableExplorationTweetsCandidatePipelineParam, EnableJetfuelFramePipelineParam, EnableBookmarksModuleWeekendGate, EnablePinnedTweetsCandidatePipelineParam, EnableEntryPointPivotParam, EnableFlipInjectionModuleCandidatePipelineParam, EnableGrokEntryPointPivotParam, EnableRecommendedJobsParam, EnableRecommendedRecruitingOrganizationsParam, EnableTweetPreviewsCandidatePipelineParam, EnableViewerHasJobRecommendationsFeatureParam, EnableWhoToFollowCandidatePipelineParam, EnableWhoToSubscribeCandidatePipelineParam, EnableTrendsParam, EnableKeywordTrendsParam, EnableArticlePreviewTextHydrationParam, EnableForYouTimelineAdsSurface, EnableScoredVideoTweetsCandidatePipelineParam, VideoCarouselEnableFooterParam, VideoCarouselAllowVerticalVideos, VideoCarouselAllowHorizontalVideos, RelevancePromptEnableParam, Navigation.PtrEnableParam, Navigation.ColdStartEnableParam, Navigation.WarmStartEnableParam, Navigation.ManualRefreshEnableParam, Navigation.NavigateEnableParam, EnableAdsDebugParam, EnableForYouAppUpsellParam, EnableForYouTopicSelectorParam, EnableTuneFeedCandidatePipelineParam, EnableFollowedGrokTopicsHydrationParam, EnableTLSHydrationParam ) override val boundedLongFSOverrides = Seq( ExplorationTweetsMaxFollowerCountParam ) override val boundedIntFSOverrides = Seq( AdsNumOrganicItemsParam, ClearCache.MinEntriesParam, ExplorationTweetsTimelinePosition, GrokPivotModuleTimelinePosition, SuperbowlModuleTimelinePosition, VideoCarouselNumTweetCandidatesToDedupeAgainstParam, VideoCarouselTimelinePosition, VideoCarouselNumCandidates, MaxCommunitiesToJoinCandidatesParam, MaxNumberExplorationTweetsParam, MaxNumberKeywordTrendsParam, MaxRecommendedJobCandidatesParam, MaxRecommendedRecruitingOrganizationCandidatesParam, ServerMaxResultsParam, TweetPreviewsMaxCandidatesParam, RelevancePromptTweetPositionParam, ForYouAppUpsellPosition, ForYouTopicSelectorPosition, TuneFeedTimelinePosition, MinFollowingCountParam, MaxFollowingCountParam ) override val stringFSOverrides = Seq( WhoToFollowDisplayLocationParam, ExperimentStatsParam, RelevancePromptTitleParam, RelevancePromptPositiveParam, RelevancePromptNegativeParam, RelevancePromptNeutralParam, ForYouAppUpsellJetfuelRouteParam, ForYouTopicSelectorJetfuelRouteParam, ) override val boundedDurationFSOverrides = Seq( CommunitiesToJoinMinInjectionIntervalParam, RecommendedJobMinInjectionIntervalParam, RecommendedRecruitingOrganizationMinInjectionIntervalParam, WhoToFollowMinInjectionIntervalParam, WhoToSubscribeMinInjectionIntervalParam, TweetPreviewsMinInjectionIntervalParam, BookmarksModuleMinInjectionIntervalParam, InNetworkExplorationTweetsMinInjectionIntervalParam, PinnedTweetsModuleMinInjectionIntervalParam, TrendsModuleMinInjectionIntervalParam, KeywordTrendsModuleMinInjectionIntervalParam, EntryPointPivotMinInjectionIntervalParam, GrokEntryPointPivotMinInjectionIntervalParam, VideoTweetsModuleMinInjectionIntervalParam, RelevancePromptMinInjectionIntervalParam ) override val enumFSOverrides = Seq( WhoToFollowDisplayTypeIdParam, WhoToSubscribeDisplayTypeIdParam, WhoToFollowUserDisplayTypeIdParam, CommunitiesToJoinDisplayTypeIdParam ) override val booleanDeciderOverrides = Seq(EnableGetTweetsFromArchiveIndex) override val longSetFSOverrides = Seq( AuthorListForStatsParam, FollowingSportsGateUsersParam ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "events-recos/events-recos-service/src/main/thrift:events-recos-thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", "src/java/com/twitter/search/common/schema/earlybird", "src/java/com/twitter/search/queryparser/query:core-query-nodes", "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", "src/thrift/com/twitter/search:earlybird-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/TweetPreviewsQueryTransformer.scala ================================================ package com.twitter.home_mixer.product.for_you.query_transformer import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.finagle.thrift.ClientId import com.twitter.finagle.tracing.Trace import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsFeature import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.search.common.ranking.{thriftscala => scr} import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant import com.twitter.search.earlybird.{thriftscala => t} import com.twitter.search.queryparser.query.Conjunction import com.twitter.search.queryparser.query.Query import com.twitter.search.queryparser.query.search.SearchOperator import javax.inject.Inject import javax.inject.Singleton @Singleton class TweetPreviewsQueryTransformer @Inject() (clientId: ClientId) extends CandidatePipelineQueryTransformer[PipelineQuery, t.EarlybirdRequest] { private val MaxPreviewTweets = 200 private val EarlybirdRelevanceTensorflowModel = "timelines_rectweet_replica" private val SinceDuration = 7.days val MetadataOptions = t.ThriftSearchResultMetadataOptions( getReferencedTweetAuthorId = true, getFromUserId = true ) override def transform(query: PipelineQuery): t.EarlybirdRequest = { val candidatePreviewCreatorIds = query.features.map(_.get(PreviewCreatorsFeature)).getOrElse(Seq.empty) val searchQuery = new Conjunction( // Include subscriber only (aka exclusive) Tweets new SearchOperator.Builder() .setType(SearchOperator.Type.FILTER) .addOperand(EarlybirdFieldConstant.EXCLUSIVE_FILTER_TERM) .build(), // Include only original Tweets new SearchOperator.Builder() .setType(SearchOperator.Type.FILTER) .addOperand(EarlybirdFieldConstant.NATIVE_RETWEETS_FILTER_TERM) .setOccur(Query.Occur.MUST_NOT) .build(), new SearchOperator.Builder() .setType(SearchOperator.Type.FILTER) .addOperand(EarlybirdFieldConstant.REPLIES_FILTER_TERM) .setOccur(Query.Occur.MUST_NOT) .build(), new SearchOperator.Builder() .setType(SearchOperator.Type.FILTER) .addOperand(EarlybirdFieldConstant.QUOTE_FILTER_TERM) .setOccur(Query.Occur.MUST_NOT) .build(), new SearchOperator(SearchOperator.Type.SINCE_TIME, SinceDuration.ago.inSeconds.toString) ) t.EarlybirdRequest( searchQuery = t.ThriftSearchQuery( serializedQuery = Some(searchQuery.serialize), fromUserIDFilter64 = Some(candidatePreviewCreatorIds), numResults = MaxPreviewTweets, rankingMode = t.ThriftSearchRankingMode.Relevance, relevanceOptions = Some( t.ThriftSearchRelevanceOptions( filterDups = true, keepDupWithHigherScore = true, proximityScoring = true, maxConsecutiveSameUser = Some(5), rankingParams = Some( scr.ThriftRankingParams( `type` = Some(scr.ThriftScoringFunctionType.TensorflowBased), selectedTensorflowModel = Some(EarlybirdRelevanceTensorflowModel), minScore = -1.0e100, applyBoosts = false, ) ), ), ), resultMetadataOptions = Some(MetadataOptions), searcherId = query.getOptionalUserId, ), getOlderResults = Some(true), // needed for archive access to older tweets clientRequestID = Some(s"${Trace.id.traceId}"), followedUserIds = Some(candidatePreviewCreatorIds.toSeq), numResultsToReturnAtRoot = Some(MaxPreviewTweets), clientId = Some(clientId.name), ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/UnifiedCandidatesQueryTransformer.scala ================================================ package com.twitter.home_mixer.product.for_you.query_transformer import com.twitter.events.recos.{thriftscala => t} import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.configapi.Param case class UnifiedCandidatesQueryTransformer( maxResultsParam: Param[Int], candidatePipelineIdentifier: CandidatePipelineIdentifier) extends CandidatePipelineQueryTransformer[ PipelineQuery, t.GetUnfiedCandidatesRequest ] { override val identifier: TransformerIdentifier = TransformerIdentifier("UnifiedCandidates") override def transform( query: PipelineQuery ): t.GetUnfiedCandidatesRequest = { t.GetUnfiedCandidatesRequest( displayLocation = t.DisplayLocation.Guide, clientId = query.clientContext.appId, userId = query.getOptionalUserId, languageCode = query.getLanguageCode, countryCode = query.getCountryCode, maxResults = Some(query.params(maxResultsParam)), userAgent = query.clientContext.userAgent, isObjectiveTrendsRequest = Some(true) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "events-recos/events-recos-service/src/main/thrift:events-recos-thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/trends_events", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", "src/thrift/com/twitter/frigate/bookmarks:bookmarks-thrift-scala", "src/thrift/com/twitter/search:earlybird-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BookmarksResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.for_you.response_transformer import com.twitter.frigate.bookmarks.thriftscala.BookmarkedTweet import com.twitter.home_mixer.model.HomeFeatures.BookmarkedTweetTimestamp import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier object BookmarksResponseFeatureTransformer extends CandidateFeatureTransformer[BookmarkedTweet] { override val identifier: TransformerIdentifier = TransformerIdentifier("BookmarksResponse") override val features: Set[Feature[_, _]] = Set(BookmarkedTweetTimestamp, ServedTypeFeature) def transform(input: BookmarkedTweet): FeatureMap = FeatureMapBuilder() .add(BookmarkedTweetTimestamp, Some(input.timestamp)) .add(ServedTypeFeature, hmt.ServedType.ForYouResurfacedBookmark) .build() } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/ExplorationTweetResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.for_you.response_transformer import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier object ExplorationTweetResponseFeatureTransformer extends CandidateFeatureTransformer[Long] { override val identifier: TransformerIdentifier = TransformerIdentifier("ExplorationTweetResponse") override val features: Set[Feature[_, _]] = Set(ServedTypeFeature) def transform( input: Long ): FeatureMap = { FeatureMapBuilder() .add(ServedTypeFeature, hmt.ServedType.ForYouExploration) .build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/KeywordTrendsFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.for_you.response_transformer import com.twitter.events.recos.{thriftscala => t} import com.twitter.home_mixer.util.UrtUtil import com.twitter.home_mixer.product.for_you.candidate_source.TrendCandidate import com.twitter.product_mixer.component_library.model.candidate.trends_events._ import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.item.trend.GroupedTrend object KeywordTrendsFeatureTransformer extends CandidateFeatureTransformer[TrendCandidate] { override val identifier: TransformerIdentifier = TransformerIdentifier("KeywordTrends") override def features: Set[Feature[_, _]] = Set( TrendNormalizedNameFeature, TrendNameFeature, TrendUrlFeature, TrendDescriptionFeature, TrendTweetCountFeature, TrendDomainContextFeature, TrendGroupedTrendsFeature, PromotedTrendNameFeature, TrendRankFeature ) override def transform(input: TrendCandidate): FeatureMap = { val trend: t.TrendCandidate = input.candidate FeatureMapBuilder() .add(TrendNameFeature, trend.trendName) .add(TrendDescriptionFeature, trend.context.flatMap(_.description)) .add(TrendNormalizedNameFeature, trend.normalizedTrendName) .add(TrendUrlFeature, UrtUtil.transformUrl(trend.url)) .add(TrendTweetCountFeature, trend.context.flatMap(_.tweetCount)) .add(TrendDomainContextFeature, trend.domainContext) .add(TrendGroupedTrendsFeature, trend.relatedTrends.map(transformGroupedTrends)) .add(PromotedTrendNameFeature, trend.promotedMetadata.map(_.name)) .add(TrendRankFeature, input.rank) .build() } private def transformGroupedTrends(groupedTrends: Seq[t.RelatedTrend]): Seq[GroupedTrend] = { groupedTrends.map { trend => GroupedTrend( trendName = trend.trendName, url = UrtUtil.transformUrl(trend.url) ) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/PinnedTweetResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.for_you.response_transformer import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.product.for_you.candidate_source.PinnedTweetCandidate import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier object PinnedTweetResponseFeatureTransformer extends CandidateFeatureTransformer[PinnedTweetCandidate] { override val identifier: TransformerIdentifier = TransformerIdentifier("PinnedTweetResponse") override val features: Set[Feature[_, _]] = Set(AuthorIdFeature, ServedTypeFeature) def transform( input: PinnedTweetCandidate ): FeatureMap = { FeatureMapBuilder() .add(AuthorIdFeature, input.userId) .add(ServedTypeFeature, hmt.ServedType.ForYouPinned) .build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/ScoredVideoTweetResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.for_you.response_transformer import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.VideoAspectRatioFeature import com.twitter.home_mixer.model.HomeFeatures.VideoDisplayTypeFeature import com.twitter.home_mixer.product.for_you.candidate_source.ScoredVideoTweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.Carousel import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.CompactCarousel object ScoredVideoTweetResponseFeatureTransformer extends CandidateFeatureTransformer[ScoredVideoTweetCandidate] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredVideoTweetResponse") override val features: Set[Feature[_, _]] = Set(AuthorIdFeature, ServedTypeFeature, VideoAspectRatioFeature, VideoDisplayTypeFeature) def transform( input: ScoredVideoTweetCandidate ): FeatureMap = { val displayType = input.aspectRatio.map { ratio => if (ratio > 1.0) Carousel else CompactCarousel } FeatureMapBuilder() .add(AuthorIdFeature, Some(input.authorId)) .add(ServedTypeFeature, input.servedType) .add(VideoAspectRatioFeature, input.aspectRatio.map(_.toFloat)) .add(VideoDisplayTypeFeature, displayType) .build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/StoriesModuleResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.for_you.response_transformer import com.twitter.home_mixer.product.for_you.candidate_source.StoryCandidate import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendDescriptionFeature import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendDomainContextFeature import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendNameFeature import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendSocialContextImages import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendThumbnail import com.twitter.product_mixer.component_library.model.candidate.trends_events.TrendUrlFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.graphql.ApiImage import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.DeepLink import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Url object StoriesModuleResponseFeatureTransformer extends CandidateFeatureTransformer[StoryCandidate] { override val identifier: TransformerIdentifier = TransformerIdentifier("StoriesModuleResponse") override val features: Set[Feature[_, _]] = Set( TrendNameFeature, TrendDescriptionFeature, TrendUrlFeature, TrendDomainContextFeature, TrendSocialContextImages, TrendThumbnail ) def transform(input: StoryCandidate): FeatureMap = { val url = input.id val thumbnail = input.thumbnail.map { img => ApiImage( height = img.originalImgHeight, width = img.originalImgWidth, url = img.originalImgUrl ) } FeatureMapBuilder() .add(TrendNameFeature, input.title) .add(TrendDescriptionFeature, None) .add(TrendUrlFeature, Url(DeepLink, url)) .add(TrendDomainContextFeature, Some(input.context)) .add(TrendSocialContextImages, Some(input.socialProof)) .add(TrendThumbnail, thumbnail) .build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TuneFeedFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.for_you.response_transformer import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier object TuneFeedFeatureTransformer extends CandidateFeatureTransformer[TweetCandidate] { override val identifier: TransformerIdentifier = TransformerIdentifier("TuneFeed") override val features: Set[Feature[_, _]] = Set(ServedTypeFeature) def transform( input: TweetCandidate ): FeatureMap = FeatureMap(ServedTypeFeature, hmt.ServedType.ForYouPopularTopic) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.for_you.response_transformer import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsTweetPreviewFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.search.earlybird.{thriftscala => eb} object TweetPreviewResponseFeatureTransformer extends CandidateFeatureTransformer[eb.ThriftSearchResult] { override val identifier: TransformerIdentifier = TransformerIdentifier("TweetPreviewResponse") override val features: Set[Feature[_, _]] = Set(AuthorIdFeature, IsTweetPreviewFeature, ServedTypeFeature) def transform( input: eb.ThriftSearchResult ): FeatureMap = { FeatureMapBuilder() .add(IsTweetPreviewFeature, true) .add(ServedTypeFeature, hmt.ServedType.ForYouTweetPreview) .add(AuthorIdFeature, input.metadata.map(_.fromUserId)) .build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/PinnedTweetCandidateScorer.scala ================================================ package com.twitter.home_mixer.product.for_you.scorer import com.twitter.home_mixer.model.HomeFeatures.AuthorFollowersFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object PinnedTweetCandidateScorer extends Scorer[PipelineQuery, TweetCandidate] { override val identifier: ScorerIdentifier = ScorerIdentifier("PinnedTweetCandidate") override def features: Set[Feature[_, _]] = Set(ScoreFeature) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { val featureMaps = candidates.map { candidate => val score = candidate.features.getOrElse(AuthorFollowersFeature, None) FeatureMap(ScoreFeature, score.map(_.toDouble)) } Stitch.value(featureMaps) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/DebugUpdateSortAdsResult.scala ================================================ package com.twitter.home_mixer.product.for_you.selector import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.functional_component.common.CandidateScope.PartitionedCandidates import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.selector.SelectorResult import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery // This selector is used for debugging Ads. Tt will put all the Ads candidates to the top case class DebugUpdateSortAdsResult( adsCandidatePipeline: CandidatePipelineIdentifier) extends Selector[PipelineQuery] { override val pipelineScope: CandidateScope = SpecificPipeline(adsCandidatePipeline) override def apply( query: PipelineQuery, remainingCandidates: Seq[CandidateWithDetails], result: Seq[CandidateWithDetails] ): SelectorResult = { val PartitionedCandidates(adCandidates, otherRemainingCandidates) = pipelineScope.partition(result) SelectorResult( remainingCandidates = remainingCandidates, result = adCandidates ++ otherRemainingCandidates) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/DebunchCandidates.scala ================================================ package com.twitter.home_mixer.product.for_you.selector import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.functional_component.common.CandidateScope.PartitionedCandidates import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.selector.SelectorResult import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.configapi.Param trait MustDebunch { def apply(candidate: CandidateWithDetails): Boolean } /** * This selector rearranges the candidates to only allow bunches of size [[maxBunchSize]], * where a bunch is a consecutive sequence of candidates that meet [[mustDebunch]]. */ case class DebunchCandidates( override val pipelineScope: CandidateScope, mustDebunch: MustDebunch, maxBunchSize: Param[Int]) extends Selector[PipelineQuery] { override def apply( query: PipelineQuery, remainingCandidates: Seq[CandidateWithDetails], result: Seq[CandidateWithDetails] ): SelectorResult = { val PartitionedCandidates(selectedCandidates, otherCandidates) = pipelineScope.partition(remainingCandidates) val mutableCandidates = collection.mutable.ListBuffer(selectedCandidates: _*) var candidatePointer = 0 var nonDebunchPointer = 0 var bunchSize = 0 var finalNonDebunch = -1 while (candidatePointer < mutableCandidates.size) { if (mustDebunch(mutableCandidates(candidatePointer))) bunchSize += 1 else { bunchSize = 0 finalNonDebunch = candidatePointer } if (bunchSize > query.params(maxBunchSize)) { nonDebunchPointer = Math.max(candidatePointer, nonDebunchPointer) while (nonDebunchPointer < mutableCandidates.size && mustDebunch(mutableCandidates(nonDebunchPointer))) { nonDebunchPointer += 1 } if (nonDebunchPointer == mutableCandidates.size) candidatePointer = mutableCandidates.size else { val nextNonDebunch = mutableCandidates(nonDebunchPointer) mutableCandidates.remove(nonDebunchPointer) mutableCandidates.insert(candidatePointer, nextNonDebunch) bunchSize = 0 finalNonDebunch = candidatePointer } } candidatePointer += 1 } val debunchedCandidates = mutableCandidates.toList val updatedCandidates = otherCandidates ++ debunchedCandidates SelectorResult(remainingCandidates = updatedCandidates, result = result) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/selector/RemoveDuplicateCandidatesOutsideModule.scala ================================================ package com.twitter.home_mixer.product.for_you.selector import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.selector.SelectorResult import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.configapi.Param case class RemoveDuplicateCandidatesOutsideModule( override val pipelineScope: CandidateScope, candidatePipelinesOutsideModule: Set[CandidatePipelineIdentifier], numCandidatesToCompareAgainst: Param[Int]) extends Selector[PipelineQuery] { override def apply( query: PipelineQuery, remainingCandidates: Seq[CandidateWithDetails], result: Seq[CandidateWithDetails] ): SelectorResult = { val candidatesOutsideModule = getCandidateIdsOutsideModule(query, remainingCandidates) val finalRemainingCandidates = remainingCandidates.map { case module: ModuleCandidateWithDetails if pipelineScope.contains(module) => module.copy(candidates = module.candidates.filterNot { candidate => candidatesOutsideModule.contains(candidate.candidateIdLong) }) case candidate => candidate } SelectorResult(remainingCandidates = finalRemainingCandidates, result = result) } private def getCandidateIdsOutsideModule( query: PipelineQuery, remainingCandidates: Seq[CandidateWithDetails] ): Set[Long] = { remainingCandidates .collect { case candidate: ItemCandidateWithDetails if candidatePipelinesOutsideModule.contains(candidate.source) => candidate.candidateIdLong }.take(query.params(numCandidatesToCompareAgainst)).toSet } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "kafka/finagle-kafka/finatra-kafka/src/main/scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", "src/scala/com/twitter/timelines/prediction/common/adapters", "src/thrift/com/twitter/timelines/served_candidates_logging:served_candidates_logging-scala", "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", "timelines/ml:kafka", "timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala ================================================ package com.twitter.home_mixer.product.for_you.side_effect import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableServedCandidateFeatureKeysKafkaPublishingParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.side_effect.KafkaPublishingSideEffect import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.ServedCandidateFeatureKeysAdapter import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.ServedCandidateFeatureKeysFields import com.twitter.timelines.ml.kafka.serde.CandidateFeatureKeySerde import com.twitter.timelines.ml.kafka.serde.TBaseSerde import com.twitter.timelines.served_candidates_logging.{thriftscala => sc} import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.common.serialization.Serializer import scala.collection.JavaConverters._ /** * Pipeline side-effect that publishes candidate keys to a Kafka topic. */ class ServedCandidateFeatureKeysKafkaSideEffect( topic: String, sourceIdentifiers: Set[identifier.CandidatePipelineIdentifier]) extends KafkaPublishingSideEffect[ sc.CandidateFeatureKey, pldr.PolyDataRecord, PipelineQuery, Timeline ] with PipelineResultSideEffect.Conditionally[PipelineQuery, Timeline] { override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedCandidateFeatureKeys") override def onlyIf( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: Timeline ): Boolean = query.params.getBoolean(EnableServedCandidateFeatureKeysKafkaPublishingParam) override val bootstrapServer: String = "/s/kafka/timeline:kafka-tls" override val keySerde: Serializer[sc.CandidateFeatureKey] = CandidateFeatureKeySerde().serializer() override val valueSerde: Serializer[pldr.PolyDataRecord] = TBaseSerde.Thrift[pldr.PolyDataRecord]().serializer override val clientId: String = "home_mixer_served_candidate_feature_keys_producer" override def buildRecords( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: Timeline ): Seq[ProducerRecord[sc.CandidateFeatureKey, pldr.PolyDataRecord]] = { query.features .flatMap(_.getOrElse(ServedIdFeature, None)) .fold(Seq.empty[ProducerRecord[sc.CandidateFeatureKey, pldr.PolyDataRecord]]) { servedId => CandidatesUtil .getItemCandidates { selectedCandidates.iterator .filter(candidate => sourceIdentifiers.contains(candidate.source)).toSeq } .flatMap { candidate => candidate.features .getOrElse(PredictionRequestIdFeature, None) .map { predictionRequestId => val key = sc.CandidateFeatureKey( tweetId = candidate.candidateIdLong, viewerId = query.getRequiredUserId, servedId = -1L) val record = ServedCandidateFeatureKeysAdapter .adaptToDataRecords(ServedCandidateFeatureKeysFields( viewerId = query.getRequiredUserId, tweetId = candidate.candidateIdLong, predictionRequestId = predictionRequestId, servedRequestIdOpt = None, servedId = servedId, injectionModuleName = candidate.getClass.getSimpleName, viewerFollowsOriginalAuthor = Some(candidate.features.getOrElse(InNetworkFeature, true)), finalPositionIndex = Some(candidate.sourcePosition), isReadFromCache = candidate.features.getOrElse(IsReadFromCacheFeature, false) )).asScala.head new ProducerRecord(topic, key, pldr.PolyDataRecord.dataRecord(record)) } } } } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(98.5) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffectBuilder.scala ================================================ package com.twitter.home_mixer.product.for_you.side_effect import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import javax.inject.Inject import javax.inject.Singleton @Singleton case class ServedCandidateFeatureKeysKafkaSideEffectBuilder @Inject() ( injectedServiceIdentifier: ServiceIdentifier) { def build( sourceIdentifiers: Set[CandidatePipelineIdentifier] ): ServedCandidateFeatureKeysKafkaSideEffect = { val topic = injectedServiceIdentifier.environment.toLowerCase match { case "prod" => "tq_ct_served_candidate_feature_keys" case _ => "tq_ct_served_candidate_feature_keys_staging" } new ServedCandidateFeatureKeysKafkaSideEffect(topic, sourceIdentifiers) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKafkaSideEffect.scala ================================================ package com.twitter.home_mixer.product.for_you.side_effect import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery object ServedCandidateKafkaSideEffect { def extractCandidates( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], sourceIdentifiers: Set[CandidatePipelineIdentifier] ): Seq[ItemCandidateWithDetails] = { val servedRequestIdOpt = query.features.getOrElse(FeatureMap.empty).getOrElse(ServedRequestIdFeature, None) selectedCandidates.iterator .filter(candidate => sourceIdentifiers.contains(candidate.source)) .flatMap { case item: ItemCandidateWithDetails => Seq(item) case module: ModuleCandidateWithDetails => module.candidates } .filter(candidate => candidate.features.getOrElse(StreamToKafkaFeature, false)) .map { candidate => val servedId = if (candidate.features.getOrElse(IsReadFromCacheFeature, false) && servedRequestIdOpt.nonEmpty) servedRequestIdOpt else candidate.features.getOrElse(PredictionRequestIdFeature, None) candidate.copy(features = candidate.features + (ServedIdFeature, servedId)) }.toSeq // deduplicate by (tweetId, userId, servedId) .groupBy { candidate => ( candidate.candidateIdLong, query.getRequiredUserId, candidate.features.getOrElse(ServedIdFeature, None)) }.values.map(_.head).toSeq } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffect.scala ================================================ package com.twitter.home_mixer.product.for_you.side_effect import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableServedCandidateKafkaPublishingParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.ml.api.DataRecord import com.twitter.ml.api.util.SRichDataRecord import com.twitter.product_mixer.component_library.side_effect.KafkaPublishingSideEffect import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.DataRecordLoggingRelatedFeatures.tlmServedKeysFeatureContext import com.twitter.timelines.ml.kafka.serde.ServedCandidateKeySerde import com.twitter.timelines.ml.kafka.serde.TBaseSerde import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import com.twitter.timelines.served_candidates_logging.{thriftscala => sc} import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} import com.twitter.util.Time import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.common.serialization.Serializer /** * Pipeline side-effect that publishes candidate keys to a Kafka topic. */ class ServedCandidateKeysKafkaSideEffect( topic: String, sourceIdentifiers: Set[CandidatePipelineIdentifier]) extends KafkaPublishingSideEffect[ sc.ServedCandidateKey, pldr.PolyDataRecord, PipelineQuery, Timeline ] with PipelineResultSideEffect.Conditionally[PipelineQuery, Timeline] { import ServedCandidateKafkaSideEffect._ override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedCandidateKeys") override def onlyIf( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: Timeline ): Boolean = query.params.getBoolean(EnableServedCandidateKafkaPublishingParam) override val bootstrapServer: String = "/s/kafka/timeline:kafka-tls" override val keySerde: Serializer[sc.ServedCandidateKey] = ServedCandidateKeySerde.serializer() override val valueSerde: Serializer[pldr.PolyDataRecord] = TBaseSerde.Thrift[pldr.PolyDataRecord]().serializer override val clientId: String = "home_mixer_served_candidate_keys_producer" override def buildRecords( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: Timeline ): Seq[ProducerRecord[sc.ServedCandidateKey, pldr.PolyDataRecord]] = { val servedTimestamp = Time.now.inMilliseconds val servedRequestIdOpt = query.features.getOrElse(FeatureMap.empty).getOrElse(ServedRequestIdFeature, None) extractCandidates(query, selectedCandidates, sourceIdentifiers).collect { // Only publish non-cached tweets to the ServedCandidateKey topic case candidate if !candidate.features.getOrElse(IsReadFromCacheFeature, false) => val key = sc.ServedCandidateKey( tweetId = candidate.candidateIdLong, viewerId = query.getRequiredUserId, servedId = -1L ) val record = SRichDataRecord(new DataRecord, tlmServedKeysFeatureContext) record.setFeatureValueFromOption( TimelinesSharedFeatures.PREDICTION_REQUEST_ID, candidate.features.getOrElse(PredictionRequestIdFeature, None) ) record .setFeatureValueFromOption(TimelinesSharedFeatures.SERVED_REQUEST_ID, servedRequestIdOpt) record.setFeatureValueFromOption( TimelinesSharedFeatures.SERVED_ID, candidate.features.getOrElse(ServedIdFeature, None) ) record.setFeatureValueFromOption( TimelinesSharedFeatures.INJECTION_TYPE, record.getFeatureValueOpt(TimelinesSharedFeatures.INJECTION_TYPE)) record.setFeatureValue( TimelinesSharedFeatures.SERVED_TIMESTAMP, servedTimestamp ) record.record.dropUnknownFeatures() new ProducerRecord(topic, key, pldr.PolyDataRecord.dataRecord(record.getRecord)) } } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(98.5) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffectBuilder.scala ================================================ package com.twitter.home_mixer.product.for_you.side_effect import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import javax.inject.Inject import javax.inject.Singleton @Singleton case class ServedCandidateKeysKafkaSideEffectBuilder @Inject() ( injectedServiceIdentifier: ServiceIdentifier) { def build( sourceIdentifiers: Set[CandidatePipelineIdentifier] ): ServedCandidateKeysKafkaSideEffect = { val topic = injectedServiceIdentifier.environment.toLowerCase match { case "prod" => "tq_ct_served_candidate_keys" case _ => "tq_ct_served_candidate_keys_staging" } new ServedCandidateKeysKafkaSideEffect(topic, sourceIdentifiers) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.scala ================================================ package com.twitter.home_mixer.product.for_you.side_effect import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.AuthorListForDataCollectionParam import com.twitter.home_mixer.product.for_you.param.ForYouParam.AuthorListForStatsParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.util.logging.Logging case class ServedStatsSideEffect( candidatePipelines: Set[CandidatePipelineIdentifier], statsReceiver: StatsReceiver) extends PipelineResultSideEffect[PipelineQuery, Timeline] with Logging { override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedStats") private val baseStatsReceiver = statsReceiver.scope(identifier.toString) private val authorStatsReceiver = baseStatsReceiver.scope("Author") private val tweetStatsReceiver = baseStatsReceiver.scope("Tweet") private val servedTypeStatsReceiver = baseStatsReceiver.scope("ServedType") private val followedUsersStatsReceiver = baseStatsReceiver.scope("FollowedUsers") private val responseSizeStatsReceiver = baseStatsReceiver.scope("ResponseSize") private val contentBalanceStatsReceiver = baseStatsReceiver.scope("ContentBalance") private val inNetworkStatsReceiver = contentBalanceStatsReceiver.scope("InNetwork") private val outOfNetworkStatsReceiver = contentBalanceStatsReceiver.scope("OutOfNetwork") private val replyStatsReceiver = contentBalanceStatsReceiver.scope("Reply") private val originalStatsReceiver = contentBalanceStatsReceiver.scope("Original") private val emptyStatsReceiver = responseSizeStatsReceiver.scope("Empty") private val lessThan5StatsReceiver = responseSizeStatsReceiver.scope("LessThan5") private val lessThan10StatsReceiver = responseSizeStatsReceiver.scope("LessThan10") override def apply( inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] ): Stitch[Unit] = { val tweetCandidates = CandidatesUtil .getItemCandidates(inputs.selectedCandidates) .filter(candidate => candidatePipelines.contains(candidate.source)) recordAuthorStats( candidates = tweetCandidates, authors = inputs.query.params(AuthorListForDataCollectionParam), tweetLevelAuthors = inputs.query.params(AuthorListForStatsParam) ) recordServedTypeStats(tweetCandidates) recordFollowedUsersStats( tweetCandidates, inputs.query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty).size).getOrElse(0), ) recordContentBalanceStats(tweetCandidates) recordResponseSizeStats(tweetCandidates) logColdContentData(tweetCandidates, inputs.query.getRequiredUserId) Stitch.Unit } def recordAuthorStats( candidates: Seq[CandidateWithDetails], authors: Set[Long], tweetLevelAuthors: Set[Long] ): Unit = { val filtered = candidates .filter { candidate => candidate.features.getOrElse(AuthorIdFeature, None).exists(authors.contains) && // Only include original tweets (!candidate.features.getOrElse(IsRetweetFeature, false)) && candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty } filtered .groupBy { candidate => (getServedType(candidate), candidate.features.get(AuthorIdFeature).get) } .foreach { case ((servedType, authorId), authorCandidates) => val authorStr = authorId.toString.takeRight(9) authorStatsReceiver.scope(authorStr).counter(servedType).incr(authorCandidates.size) } filtered.map { candidate => val authorId = candidate.features.get(AuthorIdFeature).get val authorStr = authorId.toString.takeRight(9) authorStatsReceiver.scope(authorStr).counter(candidate.candidateIdLong.toString).incr() if (tweetLevelAuthors.contains(authorId)) tweetStatsReceiver.counter(candidate.candidateIdLong.toString.takeRight(10)).incr() } } def logColdContentData(candidates: Seq[CandidateWithDetails], viewerId: Long): Unit = { candidates.foreach { candidate => val servedType = candidate.features.get(ServedTypeFeature) if (servedType == hmt.ServedType.ForYouContentExplorationDeepRetrievalI2i) { logger.info("Tier1: " + candidate.candidateIdLong + " viewerId: " + viewerId) } else if (servedType == hmt.ServedType.ForYouContentExplorationTier2DeepRetrievalI2i) { logger.info("Tier2: " + candidate.candidateIdLong + " viewerId: " + viewerId) } } } def recordServedTypeStats( candidates: Seq[ItemCandidateWithDetails], ): Unit = { candidates.groupBy(getServedType).foreach { case (servedType, servedTypeCandidates) => servedTypeStatsReceiver.counter(servedType).incr(servedTypeCandidates.size) } } def recordFollowedUsersStats( candidates: Seq[ItemCandidateWithDetails], followedUsers: Int, ): Unit = { val followedUsersScope = if (followedUsers < 10) "0-9" else if (followedUsers < 100) "10-99" else if (followedUsers < 1000) "100-999" else if (followedUsers < 10000) "1000-9999" else ">10000" candidates.groupBy(getServedType).foreach { case (servedType, servedTypeCandidates) => followedUsersStatsReceiver .scope(followedUsersScope).counter(servedType) .incr(servedTypeCandidates.size) } } def recordContentBalanceStats( candidates: Seq[ItemCandidateWithDetails], ): Unit = { val (in, oon) = candidates.partition(_.features.getOrElse(InNetworkFeature, true)) inNetworkStatsReceiver.counter().incr(in.size) outOfNetworkStatsReceiver.counter().incr(oon.size) val (reply, original) = candidates.partition(_.features.getOrElse(InReplyToTweetIdFeature, None).isDefined) replyStatsReceiver.counter().incr(reply.size) originalStatsReceiver.counter().incr(original.size) } def recordResponseSizeStats( candidates: Seq[ItemCandidateWithDetails], ): Unit = { if (candidates.size == 0) emptyStatsReceiver.counter().incr() if (candidates.size < 5) lessThan5StatsReceiver.counter().incr() if (candidates.size < 10) lessThan10StatsReceiver.counter().incr() } private def getServedType(candidate: CandidateWithDetails): String = candidate.features.get(ServedTypeFeature).name override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/VideoServedStatsSideEffect.scala ================================================ package com.twitter.home_mixer.product.for_you.side_effect import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature import com.twitter.home_mixer.model.HomeFeatures.ViralContentCreatorFeature import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch case class VideoServedStatsSideEffect( candidatePipelines: Set[CandidatePipelineIdentifier], statsReceiver: StatsReceiver) extends PipelineResultSideEffect[PipelineQuery, Timeline] { override val identifier: SideEffectIdentifier = SideEffectIdentifier("VideoServedStats") private val baseStatsReceiver = statsReceiver.scope(identifier.toString) private val videoStatsReceiver = baseStatsReceiver.scope("Video") override def apply( inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] ): Stitch[Unit] = { val tweetCandidates = CandidatesUtil .getItemCandidates(inputs.selectedCandidates) .filter(candidate => candidatePipelines.contains(candidate.source)) val clientId = inputs.query.clientContext.appId.getOrElse(0L).toString // Filter candidates to include only those with video and valid duration val videoCandidates = tweetCandidates.filter { candidate => candidate.features.getOrElse(HasVideoFeature, false) && candidate.features.get(VideoDurationMsFeature).exists(_ > 0) } statsReceiver .scope(clientId).counter("HasVideo") .incr(videoCandidates.size) recordVideoDurationStats(videoCandidates, videoStatsReceiver, clientId) recordViralContentStats(videoCandidates, videoStatsReceiver, clientId) Stitch.Unit } def recordViralContentStats( candidates: Seq[ItemCandidateWithDetails], statsReceiver: StatsReceiver, clientId: String ): Unit = { val viralContentCount = candidates.count { candidate => candidate.features.getOrElse(ViralContentCreatorFeature, false) } val viralContentInNetworkCount = candidates.count { candidate => candidate.features.getOrElse(ViralContentCreatorFeature, false) && candidate.features.getOrElse(InNetworkFeature, true) } val viralContentOutOfNetworkCount = candidates.count { candidate => candidate.features.getOrElse(ViralContentCreatorFeature, false) && !candidate.features.getOrElse(InNetworkFeature, true) } statsReceiver .scope(clientId).counter("ViralContent") .incr(viralContentCount) statsReceiver .scope(clientId) .counter("ViralContentInNetwork") .incr(viralContentInNetworkCount) statsReceiver .scope(clientId) .counter("ViralContentOutOfNetwork") .incr(viralContentOutOfNetworkCount) } def recordVideoDurationStats( candidates: Seq[ItemCandidateWithDetails], statsReceiver: StatsReceiver, clientId: String ): Unit = { val lte10Sec = candidates.count(_.features.get(VideoDurationMsFeature).exists(_ <= 10000)) val gt60Sec = candidates.count(_.features.get(VideoDurationMsFeature).exists(_ > 60000)) val bt10And60Sec = candidates.count { candidate => candidate.features.get(VideoDurationMsFeature).exists { duration => duration > 10000 && duration <= 60000 } } val bt60And120Sec = candidates.count { candidate => candidate.features.get(VideoDurationMsFeature).exists { duration => duration > 60000 && duration <= 120000 } } val bt120And180Sec = candidates.count { candidate => candidate.features.get(VideoDurationMsFeature).exists { duration => duration > 120000 && duration <= 180000 } } val gt180Sec = candidates.count(_.features.get(VideoDurationMsFeature).exists(_ > 180000)) statsReceiver .scope(clientId).counter("VideoLte10Sec").incr(lte10Sec) statsReceiver .scope(clientId).counter("VideoGt60Sec").incr(gt60Sec) statsReceiver .scope(clientId).counter("VideoBt10And60Sec") .incr(bt10And60Sec) statsReceiver .scope(clientId).counter("VideoBt60And120Sec") .incr(bt60And120Sec) statsReceiver .scope(clientId).counter("VideoBt120And180Sec") .incr(bt120And180Sec) statsReceiver .scope(clientId).counter("VideoGt180Sec").incr(gt180Sec) } override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/javax/inject:javax.inject", "finatra/inject/inject-core/src/main/scala", "finatra/inject/inject-core/src/main/scala/com/twitter/inject", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", "src/thrift/com/twitter/search:blender-scala", "src/thrift/com/twitter/timelines/render:thrift-scala", ], exports = [ "src/thrift/com/twitter/timelines/render:thrift-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users import com.twitter.home_mixer.product.list_recommended_users.candidate_source.BlenderUsersCandidateSource import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsGizmoduckValidUserFeatureHydrator import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsSGSValidUserFeatureHydrator import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.RecentListMembersFeature import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.user.UserCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.gate.EmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.search.blender.thriftscala.ThriftBlenderRequest import javax.inject.Inject import javax.inject.Singleton @Singleton class BlenderUsersCandidatePipelineConfig @Inject() ( blenderUsersCandidateSource: BlenderUsersCandidateSource, isGizmoduckValidUserFeatureHydrator: IsGizmoduckValidUserFeatureHydrator, isSGSValidUserFeatureHydrator: IsSGSValidUserFeatureHydrator) extends CandidatePipelineConfig[ ListRecommendedUsersQuery, ThriftBlenderRequest, Long, UserCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("BlenderUsers") override val gates: Seq[Gate[ListRecommendedUsersQuery]] = Seq(EmptySeqFeatureGate(RecentListMembersFeature)) override val queryTransformer: CandidatePipelineQueryTransformer[ ListRecommendedUsersQuery, ThriftBlenderRequest ] = BlenderUsersCandidatePipelineQueryTransformer override val candidateSource: BaseCandidateSource[ThriftBlenderRequest, Long] = blenderUsersCandidateSource override val resultTransformer: CandidatePipelineResultsTransformer[ Long, UserCandidate ] = { candidate => UserCandidate(id = candidate) } override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ListRecommendedUsersQuery, UserCandidate, _] ] = Seq( isGizmoduckValidUserFeatureHydrator, isSGSValidUserFeatureHydrator ) override val decorator: Option[CandidateDecorator[ListRecommendedUsersQuery, UserCandidate]] = { val clientEventInfoBuilder = ClientEventInfoBuilder("user") val userItemBuilder = UserCandidateUrtItemBuilder(clientEventInfoBuilder) Some(UrtItemCandidateDecorator(userItemBuilder)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineQueryTransformer.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.search.adaptive.adaptive_results.thriftscala.ResultType import com.twitter.search.blender.adaptive_search.thriftscala.AdaptiveSearchRequest import com.twitter.search.blender.thriftscala.ThriftBlenderRequest import com.twitter.search.blender.thriftscala.ThriftBlenderTweetypieOptions import com.twitter.search.blender.thriftscala.ThriftBlenderWorkflowID import com.twitter.search.common.constants.thriftscala.ThriftQuerySource import com.twitter.spam.rtf.thriftscala.SafetyLevel object BlenderUsersCandidatePipelineQueryTransformer extends CandidatePipelineQueryTransformer[ListRecommendedUsersQuery, ThriftBlenderRequest] { override val identifier: TransformerIdentifier = TransformerIdentifier("BlenderUsers") /** * This is a user-defined descriptor used by Blender to track the source of traffic, and it * is different from a client id, which is set during Finagle client construction. */ private val ClientAppName = "timelinemixer.list_recommended_users" override def transform(query: ListRecommendedUsersQuery): ThriftBlenderRequest = { ThriftBlenderRequest( workflowID = Some(ThriftBlenderWorkflowID.AdaptiveSearch), userID = Some(query.getRequiredUserId), // perspectival uiLang = query.clientContext.languageCode, // perspectival clientAppName = Some(ClientAppName), adaptiveSearchRequest = Some( AdaptiveSearchRequest( rawQuery = query.listName, numResults = 40, getPromotedContent = false, resultFilter = Some(ResultType.User), ) ), querySource = Some(ThriftQuerySource.TypedQuery), getCorrections = true, tweetypieOptions = Some( ThriftBlenderTweetypieOptions( safetyLevel = Some(SafetyLevel.Recommendations) ) ) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users import com.twitter.hermit.candidate.{thriftscala => t} import com.twitter.home_mixer.product.list_recommended_users.candidate_source.SimilarityBasedUsersCandidateSource import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsGizmoduckValidUserFeatureHydrator import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsListMemberFeatureHydrator import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsSGSValidUserFeatureHydrator import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.RecentListMembersFeature import com.twitter.home_mixer.product.list_recommended_users.filter.DropMaxCandidatesByAggregatedScoreFilter import com.twitter.home_mixer.product.list_recommended_users.filter.PreviouslyServedUsersFilter import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsListMemberFeature import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.user.UserCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import javax.inject.Inject import javax.inject.Singleton @Singleton class ListMemberBasedUsersCandidatePipelineConfig @Inject() ( similarityBasedUsersCandidateSource: SimilarityBasedUsersCandidateSource, isGizmoduckValidUserFeatureHydrator: IsGizmoduckValidUserFeatureHydrator, isListMemberFeatureHydrator: IsListMemberFeatureHydrator, isSGSValidUserFeatureHydrator: IsSGSValidUserFeatureHydrator) extends CandidatePipelineConfig[ ListRecommendedUsersQuery, Seq[Long], t.Candidate, UserCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ListMemberBasedUsers") override val gates: Seq[Gate[ListRecommendedUsersQuery]] = Seq(NonEmptySeqFeatureGate(RecentListMembersFeature)) override val queryTransformer: CandidatePipelineQueryTransformer[ListRecommendedUsersQuery, Seq[ Long ]] = { query => query.features.map(_.getOrElse(RecentListMembersFeature, Seq.empty)).getOrElse(Seq.empty) } override val candidateSource: BaseCandidateSource[Seq[Long], t.Candidate] = similarityBasedUsersCandidateSource override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[t.Candidate] ] = Seq(ListMemberBasedUsersResponseFeatureTransfromer) override val resultTransformer: CandidatePipelineResultsTransformer[ t.Candidate, UserCandidate ] = { candidate => UserCandidate(id = candidate.userId) } override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ListRecommendedUsersQuery, UserCandidate, _] ] = Seq(isListMemberFeatureHydrator) override val filters: Seq[Filter[ListRecommendedUsersQuery, UserCandidate]] = Seq( PreviouslyServedUsersFilter, PredicateFeatureFilter.fromPredicate( FilterIdentifier("IsListMember"), shouldKeepCandidate = { features => !features.getOrElse(IsListMemberFeature, false) } ), DropMaxCandidatesByAggregatedScoreFilter ) override val postFilterFeatureHydration: Seq[ BaseCandidateFeatureHydrator[ListRecommendedUsersQuery, UserCandidate, _] ] = Seq( isGizmoduckValidUserFeatureHydrator, isSGSValidUserFeatureHydrator ) override val decorator: Option[CandidateDecorator[ListRecommendedUsersQuery, UserCandidate]] = { val clientEventInfoBuilder = ClientEventInfoBuilder("user") val userItemBuilder = UserCandidateUrtItemBuilder(clientEventInfoBuilder) Some(UrtItemCandidateDecorator(userItemBuilder)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersResponseFeatureTransfromer.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users import com.twitter.hermit.candidate.{thriftscala => t} import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.ScoreFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier object ListMemberBasedUsersResponseFeatureTransfromer extends CandidateFeatureTransformer[t.Candidate] { override val identifier: TransformerIdentifier = TransformerIdentifier("ListMemberBasedUsers") override val features: Set[Feature[_, _]] = Set(ScoreFeature) override def transform(candidate: t.Candidate): FeatureMap = FeatureMapBuilder() .add(ScoreFeature, candidate.score) .build() } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersMixerPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.RecentListMembersQueryFeatureHydrator import com.twitter.home_mixer.product.list_recommended_users.gate.ViewerIsListOwnerGate import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsGizmoduckValidUserFeature import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsSGSValidUserFeature import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ExcludedIdsMaxLengthParam import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ServerMaxResultsParam import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AddEntriesWithReplaceInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UnorderedExcludeIdsBottomCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder import com.twitter.product_mixer.component_library.selector.DropFilteredCandidates import com.twitter.product_mixer.component_library.selector.DropMaxCandidates import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig import com.twitter.timelines.render.{thriftscala => urt} import javax.inject.Inject import javax.inject.Singleton @Singleton class ListRecommendedUsersMixerPipelineConfig @Inject() ( listMemberBasedUsersCandidatePipelineConfig: ListMemberBasedUsersCandidatePipelineConfig, blenderUsersCandidatePipelineConfig: BlenderUsersCandidatePipelineConfig, viewerIsListOwnerGate: ViewerIsListOwnerGate, recentListMembersQueryFeatureHydrator: RecentListMembersQueryFeatureHydrator, urtTransportMarshaller: UrtTransportMarshaller) extends MixerPipelineConfig[ListRecommendedUsersQuery, Timeline, urt.TimelineResponse] { override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ListRecommendedUsers") override val gates = Seq(viewerIsListOwnerGate) override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ListRecommendedUsersQuery]] = Seq(recentListMembersQueryFeatureHydrator) override val candidatePipelines: Seq[ CandidatePipelineConfig[ListRecommendedUsersQuery, _, _, _] ] = Seq( listMemberBasedUsersCandidatePipelineConfig, blenderUsersCandidatePipelineConfig ) private val candidatePipelineIdentifiers = Set( listMemberBasedUsersCandidatePipelineConfig.identifier, blenderUsersCandidatePipelineConfig.identifier ) override val resultSelectors: Seq[Selector[ListRecommendedUsersQuery]] = Seq( DropFilteredCandidates( candidatePipelines = candidatePipelineIdentifiers, filter = candidate => candidate.features.getOrElse(IsSGSValidUserFeature, false) && candidate.features.getOrElse(IsGizmoduckValidUserFeature, false) ), DropMaxCandidates( candidatePipelines = candidatePipelineIdentifiers, maxSelectionsParam = ServerMaxResultsParam), InsertAppendResults(candidatePipelineIdentifiers) ) override val domainMarshaller: DomainMarshaller[ListRecommendedUsersQuery, Timeline] = { val instructionBuilders = Seq( ReplaceEntryInstructionBuilder(ReplaceAllEntries), AddEntriesWithReplaceInstructionBuilder() ) val metadataBuilder = UrtMetadataBuilder( title = None, scribeConfigBuilder = Some( StaticTimelineScribeConfigBuilder( TimelineScribeConfig( page = Some("list_recommended_users"), section = None, entityToken = None))) ) val excludeIdsSelector: PartialFunction[UniversalNoun[_], Long] = { case item: UserItem => item.id } val cursorBuilder = UnorderedExcludeIdsBottomCursorBuilder( excludedIdsMaxLengthParam = ExcludedIdsMaxLengthParam, excludeIdsSelector = excludeIdsSelector) UrtDomainMarshaller( instructionBuilders = instructionBuilders, metadataBuilder = Some(metadataBuilder), cursorBuilders = Seq(cursorBuilder) ) } override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = urtTransportMarshaller } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersProductPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.marshaller.timelines.RecommendedUsersCursorUnmarshaller import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct import com.twitter.home_mixer.model.request.ListRecommendedUsersProductContext import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ServerMaxResultsParam import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParamConfig import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove import com.twitter.product_mixer.core.functional_component.common.alert.Alert import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert import com.twitter.product_mixer.core.functional_component.common.alert.P99 import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.request import com.twitter.product_mixer.core.pipeline.PipelineConfig import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig import com.twitter.product_mixer.core.product.ProductParamConfig import com.twitter.timelines.configapi.Params import com.twitter.timelines.render.{thriftscala => urt} import com.twitter.timelines.util.RequestCursorSerializer import com.twitter.util.Try import javax.inject.Inject import javax.inject.Singleton @Singleton class ListRecommendedUsersProductPipelineConfig @Inject() ( listRecommendedUsersMixerPipelineConfig: ListRecommendedUsersMixerPipelineConfig, listRecommendedUsersParamConfig: ListRecommendedUsersParamConfig) extends ProductPipelineConfig[ HomeMixerRequest, ListRecommendedUsersQuery, urt.TimelineResponse ] { override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("ListRecommendedUsers") override val product: request.Product = ListRecommendedUsersProduct override val paramConfig: ProductParamConfig = listRecommendedUsersParamConfig override def pipelineQueryTransformer( request: HomeMixerRequest, params: Params ): ListRecommendedUsersQuery = { val context = request.productContext match { case Some(context: ListRecommendedUsersProductContext) => context case _ => throw PipelineFailure(BadRequest, "ListRecommendedUsersProductContext not found") } val debugOptions = request.debugParams.flatMap(_.debugOptions) val pipelineCursor = request.serializedRequestCursor.flatMap { cursor => Try(UrtCursorSerializer.deserializeUnorderedExcludeIdsCursor(cursor)) .getOrElse(RecommendedUsersCursorUnmarshaller(RequestCursorSerializer.deserialize(cursor))) } ListRecommendedUsersQuery( listId = context.listId, params = params, clientContext = request.clientContext, features = None, pipelineCursor = pipelineCursor, requestedMaxResults = Some(params(ServerMaxResultsParam)), debugOptions = debugOptions, selectedUserIds = context.selectedUserIds, excludedUserIds = context.excludedUserIds, listName = context.listName ) } override def pipelines: Seq[PipelineConfig] = Seq(listRecommendedUsersMixerPipelineConfig) override def pipelineSelector(query: ListRecommendedUsersQuery): ComponentIdentifier = listRecommendedUsersMixerPipelineConfig.identifier override val alerts: Seq[Alert] = Seq( SuccessRateAlert( notificationGroup = DefaultNotificationGroup, warnPredicate = TriggerIfBelow(99.9, 20, 30), criticalPredicate = TriggerIfBelow(99.9, 30, 30), ), LatencyAlert( notificationGroup = DefaultNotificationGroup, percentile = P99, warnPredicate = TriggerIfLatencyAbove(1000.millis, 15, 30), criticalPredicate = TriggerIfLatencyAbove(1500.millis, 15, 30) ) ) override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", "src/thrift/com/twitter/search:blender-scala", "strato/config/columns/recommendations/similarity:similarity-strato-client", "strato/src/main/scala/com/twitter/strato/client", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BlenderUsersCandidateSource.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users.candidate_source import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.search.adaptive.adaptive_results.thriftscala.AdaptiveSearchResultData import com.twitter.search.adaptive.adaptive_results.thriftscala.Result import com.twitter.search.adaptive.adaptive_results.thriftscala.ResultData import com.twitter.search.blender.adaptive_search.thriftscala.AdaptiveSearchResponse import com.twitter.search.blender.adaptive_search.thriftscala.Container import com.twitter.search.blender.thriftscala.BlenderService import com.twitter.search.blender.thriftscala.ThriftBlenderRequest import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class BlenderUsersCandidateSource @Inject() ( blenderClient: BlenderService.MethodPerEndpoint) extends CandidateSource[ThriftBlenderRequest, Long] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("BlenderUsers") override def apply(request: ThriftBlenderRequest): Stitch[Seq[Long]] = { Stitch.callFuture( blenderClient.serveV2(request).map { response => val userIdsOpt = response.adaptiveSearchResponse.map(extractUserIdsFromAdaptiveSearchResponse) userIdsOpt.getOrElse(Seq.empty) } ) } private def extractUserIdsFromAdaptiveSearchResponse( response: AdaptiveSearchResponse ): Seq[Long] = { response match { case AdaptiveSearchResponse(Some(Seq(Container(Some(results), _))), _, _) => results.map(_.data).collect { case AdaptiveSearchResultData.Result(Result(ResultData.User(user), _)) => user.id } case _ => Seq.empty } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/SimilarityBasedUsersCandidateSource.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users.candidate_source import com.twitter.hermit.candidate.{thriftscala => t} import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.strato.generated.client.recommendations.similarity.SimilarUsersBySimsOnUserClientColumn import javax.inject.Inject import javax.inject.Singleton @Singleton class SimilarityBasedUsersCandidateSource @Inject() ( similarUsersBySimsOnUserClientColumn: SimilarUsersBySimsOnUserClientColumn) extends CandidateSource[Seq[Long], t.Candidate] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("SimilarityBasedUsers") private val fetcher: Fetcher[Long, Unit, t.Candidates] = similarUsersBySimsOnUserClientColumn.fetcher private val MaxCandidatesToKeep = 4000 override def apply(request: Seq[Long]): Stitch[Seq[t.Candidate]] = { Stitch .collect { request.map { userId => fetcher .fetch(userId, Unit).map { result => result.v.map(_.candidates).getOrElse(Seq.empty) }.map { candidates => val sortedCandidates = candidates.sortBy(-_.score) sortedCandidates.take(MaxCandidatesToKeep) } } }.map(_.flatten) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", "src/thrift/com/twitter/gizmoduck:thrift-scala", "src/thrift/com/twitter/socialgraph:thrift-scala", "stitch/stitch-gizmoduck", "stitch/stitch-socialgraph", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsGizmoduckValidUserFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator import com.twitter.gizmoduck.{thriftscala => gt} import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsGizmoduckValidUserFeature import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.spam.rtf.{thriftscala => rtf} import com.twitter.stitch.Stitch import com.twitter.stitch.gizmoduck.Gizmoduck import com.twitter.util.Return import javax.inject.Inject import javax.inject.Singleton @Singleton class IsGizmoduckValidUserFeatureHydrator @Inject() (gizmoduck: Gizmoduck) extends BulkCandidateFeatureHydrator[PipelineQuery, UserCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("IsGizmoduckValidUser") override val features: Set[Feature[_, _]] = Set(IsGizmoduckValidUserFeature) private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[UserCandidate]] ): Stitch[Seq[FeatureMap]] = { val context = gt.LookupContext( forUserId = query.getOptionalUserId, includeProtected = true, safetyLevel = Some(rtf.SafetyLevel.Recommendations) ) val userIds = candidates.map(_.candidate.id) Stitch .collectToTry( userIds.map(userId => gizmoduck.getUserById(userId, queryFields, context))).map { userResults => val idToUserSafetyMap = userResults .collect { case Return(user) => user }.map(user => user.id -> user.safety).toMap candidates.map { candidate => val safety = idToUserSafetyMap.getOrElse(candidate.candidate.id, None) val isValidUser = safety.isDefined && !safety.exists(_.deactivated) && !safety.exists(_.suspended) && !safety.exists(_.isProtected) && !safety.flatMap(_.offboarded).getOrElse(false) FeatureMapBuilder() .add(IsGizmoduckValidUserFeature, isValidUser) .build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsListMemberFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator import com.twitter.home_mixer.model.request.HasListId import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsListMemberFeature import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.socialgraph.{thriftscala => sg} import com.twitter.stitch.Stitch import com.twitter.stitch.socialgraph.SocialGraph import javax.inject.Inject import javax.inject.Singleton @Singleton class IsListMemberFeatureHydrator @Inject() (socialGraph: SocialGraph) extends BulkCandidateFeatureHydrator[PipelineQuery with HasListId, UserCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("IsListMember") override val features: Set[Feature[_, _]] = Set(IsListMemberFeature) override def apply( query: PipelineQuery with HasListId, candidates: Seq[CandidateWithFeatures[UserCandidate]] ): Stitch[Seq[FeatureMap]] = { val userIds = candidates.map(_.candidate.id) val request = sg.IdsRequest( relationships = Seq( sg.SrcRelationship( source = query.listId, relationshipType = sg.RelationshipType.ListHasMember, hasRelationship = true, targets = Some(userIds))), pageRequest = Some(sg.PageRequest(selectAll = Some(true))) ) socialGraph.ids(request).map(_.ids).map { listMembers => val listMembersSet = listMembers.toSet candidates.map { candidate => FeatureMapBuilder() .add(IsListMemberFeature, listMembersSet.contains(candidate.candidate.id)) .build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsSGSValidUserFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator import com.twitter.home_mixer.model.request.HasListId import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsSGSValidUserFeature import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.socialgraph.{thriftscala => sg} import com.twitter.stitch.Stitch import com.twitter.stitch.socialgraph.SocialGraph import javax.inject.Inject import javax.inject.Singleton @Singleton class IsSGSValidUserFeatureHydrator @Inject() (socialGraph: SocialGraph) extends BulkCandidateFeatureHydrator[PipelineQuery with HasListId, UserCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("IsSGSValidUser") override def features: Set[Feature[_, _]] = Set(IsSGSValidUserFeature) override def apply( query: PipelineQuery with HasListId, candidates: Seq[CandidateWithFeatures[UserCandidate]] ): Stitch[Seq[FeatureMap]] = { val sourceId = query.getRequiredUserId val targetUserIds = candidates.map(_.candidate.id) val request = sg.IdsRequest( relationships = Seq( sg.SrcRelationship( source = sourceId, relationshipType = sg.RelationshipType.Blocking, hasRelationship = true, targets = Some(targetUserIds)), sg.SrcRelationship( source = sourceId, relationshipType = sg.RelationshipType.BlockedBy, hasRelationship = true, targets = Some(targetUserIds)), sg.SrcRelationship( source = sourceId, relationshipType = sg.RelationshipType.Muting, hasRelationship = true, targets = Some(targetUserIds)) ), pageRequest = Some(sg.PageRequest(selectAll = Some(true))), context = Some(sg.LookupContext(performUnion = Some(true))) ) socialGraph.ids(request).map(_.ids).map(_.toSet).map { hasRelationshipUserIds => candidates.map { candidate => FeatureMapBuilder() .add(IsSGSValidUserFeature, !hasRelationshipUserIds.contains(candidate.candidate.id)) .build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/RecentListMembersQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator import com.twitter.home_mixer.model.request.HasListId import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.socialgraph.{thriftscala => sg} import com.twitter.stitch.Stitch import com.twitter.stitch.socialgraph.SocialGraph import javax.inject.Inject import javax.inject.Singleton case object RecentListMembersFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { override val defaultValue: Seq[Long] = Seq.empty } @Singleton class RecentListMembersQueryFeatureHydrator @Inject() (socialGraph: SocialGraph) extends QueryFeatureHydrator[PipelineQuery with HasListId] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RecentListMembers") override val features: Set[Feature[_, _]] = Set(RecentListMembersFeature) private val MaxRecentMembers = 10 override def hydrate(query: PipelineQuery with HasListId): Stitch[FeatureMap] = { val request = sg.IdsRequest( relationships = Seq(sg .SrcRelationship(query.listId, sg.RelationshipType.ListHasMember, hasRelationship = true)), pageRequest = Some(sg.PageRequest(selectAll = Some(true), count = Some(MaxRecentMembers))) ) socialGraph.ids(request).map(_.ids).map { listMembers => FeatureMapBuilder().add(RecentListMembersFeature, listMembers).build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", ], exports = [], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/DropMaxCandidatesByAggregatedScoreFilter.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users.filter import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.ScoreFeature import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object DropMaxCandidatesByAggregatedScoreFilter extends Filter[PipelineQuery, UserCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("DropMaxCandidatesByAggregatedScore") private val MaxSimilarUserCandidates = 150 override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[UserCandidate]] ): Stitch[FilterResult[UserCandidate]] = { val userIdToAggregatedScoreMap = candidates .groupBy(_.candidate.id) .map { case (userId, candidates) => val aggregatedScore = candidates.map(_.features.getOrElse(ScoreFeature, 0.0)).sum (userId, aggregatedScore) } val sortedCandidates = candidates.sortBy(candidate => -userIdToAggregatedScoreMap.getOrElse(candidate.candidate.id, 0.0)) val (kept, removed) = sortedCandidates.map(_.candidate).splitAt(MaxSimilarUserCandidates) Stitch.value(FilterResult(kept, removed)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/PreviouslyServedUsersFilter.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users.filter import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.RecentListMembersFeature import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.stitch.Stitch object PreviouslyServedUsersFilter extends Filter[ListRecommendedUsersQuery, UserCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedUsers") override def apply( query: ListRecommendedUsersQuery, candidates: Seq[CandidateWithFeatures[UserCandidate]] ): Stitch[FilterResult[UserCandidate]] = { val recentListMembers = query.features.map(_.getOrElse(RecentListMembersFeature, Seq.empty)) val servedUserIds = query.pipelineCursor.map(_.excludedIds) val excludedUserIds = (recentListMembers.getOrElse(Seq.empty) ++ query.selectedUserIds.getOrElse(Seq.empty) ++ query.excludedUserIds.getOrElse(Seq.empty) ++ servedUserIds.getOrElse(Seq.empty)).toSet val (removed, kept) = candidates.map(_.candidate).partition(candidate => excludedUserIds.contains(candidate.id)) Stitch.value(FilterResult(kept = kept, removed = removed)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", "src/thrift/com/twitter/socialgraph:thrift-scala", "stitch/stitch-socialgraph", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/ViewerIsListOwnerGate.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users.gate import com.twitter.home_mixer.model.request.HasListId import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.socialgraph.{thriftscala => sg} import com.twitter.stitch.Stitch import com.twitter.stitch.socialgraph.SocialGraph import javax.inject.Inject import javax.inject.Singleton @Singleton case class ViewerIsListOwnerGate @Inject() (socialGraph: SocialGraph) extends Gate[PipelineQuery with HasListId] { override val identifier: GateIdentifier = GateIdentifier("ViewerIsListOwner") private val relationship = sg.Relationship(relationshipType = sg.RelationshipType.ListOwning) override def shouldContinue(query: PipelineQuery with HasListId): Stitch[Boolean] = { val request = sg.ExistsRequest( source = query.getRequiredUserId, target = query.listId, relationships = Seq(relationship)) socialGraph.exists(request).map(_.exists) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", ], exports = [ "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersFeatures.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users.model import com.twitter.product_mixer.component_library.model.candidate.UserCandidate import com.twitter.product_mixer.core.feature.Feature object ListRecommendedUsersFeatures { // Candidate features object IsGizmoduckValidUserFeature extends Feature[UserCandidate, Boolean] object IsListMemberFeature extends Feature[UserCandidate, Boolean] object IsSGSValidUserFeature extends Feature[UserCandidate, Boolean] object ScoreFeature extends Feature[UserCandidate, Double] } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersQuery.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users.model import com.twitter.home_mixer.model.request.HasListId import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct import com.twitter.product_mixer.component_library.model.cursor.UrtUnorderedExcludeIdsCursor import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.request._ import com.twitter.product_mixer.core.pipeline.HasPipelineCursor import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.configapi.Params case class ListRecommendedUsersQuery( override val listId: Long, override val params: Params, override val clientContext: ClientContext, override val pipelineCursor: Option[UrtUnorderedExcludeIdsCursor], override val requestedMaxResults: Option[Int], override val debugOptions: Option[DebugOptions], override val features: Option[FeatureMap], selectedUserIds: Option[Seq[Long]], excludedUserIds: Option[Seq[Long]], listName: Option[String]) extends PipelineQuery with HasPipelineCursor[UrtUnorderedExcludeIdsCursor] with HasListId { override val product: Product = ListRecommendedUsersProduct override def withFeatureMap(features: FeatureMap): ListRecommendedUsersQuery = copy(features = Some(features)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/javax/inject:javax.inject", "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParam.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users.param import com.twitter.timelines.configapi.FSBoundedParam object ListRecommendedUsersParam { val SupportedClientFSName = "list_recommended_users_supported_client" object ServerMaxResultsParam extends FSBoundedParam[Int]( name = "list_recommended_users_server_max_results", default = 10, min = 1, max = 500 ) object ExcludedIdsMaxLengthParam extends FSBoundedParam[Int]( name = "list_recommended_users_excluded_ids_max_length", default = 2000, min = 0, max = 5000 ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParamConfig.scala ================================================ package com.twitter.home_mixer.product.list_recommended_users.param import com.twitter.home_mixer.param.decider.DeciderKey import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ExcludedIdsMaxLengthParam import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ServerMaxResultsParam import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.SupportedClientFSName import com.twitter.product_mixer.core.product.ProductParamConfig import com.twitter.servo.decider.DeciderKeyName import javax.inject.Inject import javax.inject.Singleton @Singleton class ListRecommendedUsersParamConfig @Inject() () extends ProductParamConfig { override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableListRecommendedUsersProduct override val supportedClientFSName: String = SupportedClientFSName override val boundedIntFSOverrides = Seq( ServerMaxResultsParam, ExcludedIdsMaxLengthParam ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/javax/inject:javax.inject", "ads-injection/lib/src/main/scala/com/twitter/goldfinch/api", "finatra/inject/inject-core/src/main/scala", "finatra/inject/inject-core/src/main/scala/com/twitter/inject", "home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", "src/thrift/com/twitter/timelines/render:thrift-scala", "timelines/src/main/scala/com/twitter/timelines/injection/scribe", ], exports = [ "src/thrift/com/twitter/timelines/render:thrift-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsAdsCandidatePipelineBuilder.scala ================================================ package com.twitter.home_mixer.product.list_tweets import com.twitter.adserver.{thriftscala => ads} import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate import com.twitter.home_mixer.param.HomeGlobalParams import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam import com.twitter.home_mixer.product.list_tweets.model.ListTweetsQuery import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.EnableAdsCandidatePipelineParam import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad.AdsCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfig import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfigBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.ads.CountCandidatesFromPipelines import com.twitter.product_mixer.component_library.pipeline.candidate.ads.StaticAdsDisplayLocationBuilder import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.gate.ParamNotGate import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext import com.twitter.timelines.injection.scribe.InjectionScribeUtil import com.twitter.timelineservice.suggests.{thriftscala => st} import javax.inject.Inject import javax.inject.Singleton @Singleton class ListTweetsAdsCandidatePipelineBuilder @Inject() ( adsCandidatePipelineConfigBuilder: AdsDependentCandidatePipelineConfigBuilder, adsCandidateSource: AdsProdThriftCandidateSource, advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ ListTweetsQuery, AdsCandidate ]) { private val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ListTweetsAds") private val suggestType = st.SuggestType.Promoted private val clientEventInfoBuilder = ClientEventInfoBuilder( component = InjectionScribeUtil.scribeComponent(suggestType).get, detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(suggestType.name))) ) private val contextualTweetRefBuilder = ContextualTweetRefBuilder( TweetHydrationContext( safetyLevelOverride = Some(TimelineHomePromotedHydrationSafetyLevel), outerTweetContext = None )) private val decorator = UrtItemCandidateDecorator( AdsCandidateUrtItemBuilder( tweetClientEventInfoBuilder = Some(clientEventInfoBuilder), contextualTweetRefBuilder = Some(contextualTweetRefBuilder) ) ) def build( organicCandidatePipelines: CandidateScope ): AdsDependentCandidatePipelineConfig[ListTweetsQuery] = adsCandidatePipelineConfigBuilder.build[ListTweetsQuery]( adsCandidateSource = adsCandidateSource, identifier = identifier, adsDisplayLocationBuilder = StaticAdsDisplayLocationBuilder(ads.DisplayLocation.TimelineHomeReverseChron), countNumOrganicItems = CountCandidatesFromPipelines(organicCandidatePipelines), supportedClientParam = Some(EnableAdsCandidatePipelineParam), gates = Seq( ParamNotGate( name = "AdsDisableInjectionBasedOnUserRole", param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam ), ExcludeSoftUserGate, NonEmptyCandidatesGate(organicCandidatePipelines) ), filters = Seq(ValidAdImpressionIdFilter), postFilterFeatureHydration = Seq( ParamGatedCandidateFeatureHydrator( EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, advertiserBrandSafetySettingsFeatureHydrator ) ), decorator = Some(decorator), urtRequest = Some(true), ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsMixerPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.list_tweets import com.twitter.clientapp.{thriftscala => ca} import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas import com.twitter.home_mixer.candidate_pipeline.ConversationServiceCandidatePipelineConfigBuilder import com.twitter.home_mixer.functional_component.feature_hydrator.RequestQueryFeatureHydrator import com.twitter.home_mixer.functional_component.side_effect.HomeScribeClientEventSideEffect import com.twitter.home_mixer.model.GapIncludeInstruction import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag import com.twitter.home_mixer.product.list_tweets.decorator.ListConversationServiceCandidateDecorator import com.twitter.home_mixer.product.list_tweets.model.ListTweetsQuery import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AddEntriesWithReplaceAndShowAlertInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedGapCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates import com.twitter.product_mixer.component_library.selector.ads.AdsInjector import com.twitter.product_mixer.component_library.selector.ads.InsertAdResults import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem import com.twitter.product_mixer.core.pipeline.FailOpenPolicy import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig import com.twitter.timelines.render.{thriftscala => urt} import javax.inject.Inject import javax.inject.Singleton @Singleton class ListTweetsMixerPipelineConfig @Inject() ( listTweetsTimelineServiceCandidatePipelineConfig: ListTweetsTimelineServiceCandidatePipelineConfig, conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[ ListTweetsQuery ], listTweetsAdsCandidatePipelineBuilder: ListTweetsAdsCandidatePipelineBuilder, requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ListTweetsQuery], sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, adsInjector: AdsInjector, clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], urtTransportMarshaller: UrtTransportMarshaller, @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) extends MixerPipelineConfig[ListTweetsQuery, Timeline, urt.TimelineResponse] { override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ListTweets") private val conversationServiceCandidatePipelineConfig = conversationServiceCandidatePipelineConfigBuilder.build( Seq( NonEmptyCandidatesGate( SpecificPipelines(listTweetsTimelineServiceCandidatePipelineConfig.identifier)) ), ListConversationServiceCandidateDecorator() ) private val listTweetsAdsCandidatePipelineConfig = listTweetsAdsCandidatePipelineBuilder.build( SpecificPipelines(listTweetsTimelineServiceCandidatePipelineConfig.identifier) ) override val candidatePipelines: Seq[CandidatePipelineConfig[ListTweetsQuery, _, _, _]] = Seq(listTweetsTimelineServiceCandidatePipelineConfig) override val dependentCandidatePipelines: Seq[ DependentCandidatePipelineConfig[ListTweetsQuery, _, _, _] ] = Seq(conversationServiceCandidatePipelineConfig, listTweetsAdsCandidatePipelineConfig) override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( conversationServiceCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, listTweetsAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always) override val resultSelectors: Seq[Selector[ListTweetsQuery]] = Seq( UpdateSortCandidates( ordering = CandidatesUtil.reverseChronTweetsOrdering, candidatePipeline = conversationServiceCandidatePipelineConfig.identifier ), InsertAppendResults(candidatePipeline = conversationServiceCandidatePipelineConfig.identifier), InsertAdResults( surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), adsCandidatePipeline = listTweetsAdsCandidatePipelineConfig.identifier ), ) override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ListTweetsQuery]] = Seq( requestQueryFeatureHydrator, sgsFollowedUsersQueryFeatureHydrator ) private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( enableScribeClientEvents = enableScribeClientEvents, logPipelinePublisher = clientEventsScribeEventPublisher, injectedTweetsCandidatePipelineIdentifiers = Seq(conversationServiceCandidatePipelineConfig.identifier), adsCandidatePipelineIdentifier = Some(listTweetsAdsCandidatePipelineConfig.identifier), ) override val resultSideEffects: Seq[PipelineResultSideEffect[ListTweetsQuery, Timeline]] = Seq(homeScribeClientEventSideEffect) override val domainMarshaller: DomainMarshaller[ListTweetsQuery, Timeline] = { val instructionBuilders = Seq( ReplaceEntryInstructionBuilder(ReplaceAllEntries), AddEntriesWithReplaceAndShowAlertInstructionBuilder(), ShowAlertInstructionBuilder() ) val idSelector: PartialFunction[UniversalNoun[_], Long] = { // exclude ads while determining tweet cursor values case item: TweetItem if item.promotedMetadata.isEmpty => item.id case module: TimelineModule if module.items.headOption.exists(_.item.isInstanceOf[TweetItem]) => module.items.last.item match { case item: TweetItem => item.id } } val topCursorBuilder = OrderedTopCursorBuilder(idSelector) val bottomCursorBuilder = OrderedBottomCursorBuilder(idSelector, GapIncludeInstruction.inverse()) val gapCursorBuilder = OrderedGapCursorBuilder(idSelector, GapIncludeInstruction) val metadataBuilder = UrtMetadataBuilder( title = None, scribeConfigBuilder = Some( StaticTimelineScribeConfigBuilder( TimelineScribeConfig(page = Some("list_tweets"), section = None, entityToken = None))) ) UrtDomainMarshaller( instructionBuilders = instructionBuilders, metadataBuilder = Some(metadataBuilder), cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder, gapCursorBuilder) ) } override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = urtTransportMarshaller } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsProductPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.list_tweets import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.marshaller.timelines.ChronologicalCursorUnmarshaller import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.model.request.ListTweetsProduct import com.twitter.home_mixer.model.request.ListTweetsProductContext import com.twitter.home_mixer.product.list_tweets.model.ListTweetsQuery import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.ServerMaxResultsParam import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParamConfig import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy import com.twitter.product_mixer.core.functional_component.common.alert.Alert import com.twitter.product_mixer.core.functional_component.common.alert.EmptyResponseRateAlert import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert import com.twitter.product_mixer.core.functional_component.common.alert.P99 import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert import com.twitter.product_mixer.core.functional_component.common.alert.ThroughputAlert import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.request import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor import com.twitter.product_mixer.core.pipeline.PipelineConfig import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig import com.twitter.product_mixer.core.product.ProductParamConfig import com.twitter.product_mixer.core.util.SortIndexBuilder import com.twitter.timelines.configapi.Params import com.twitter.timelines.render.{thriftscala => urt} import com.twitter.timelines.util.RequestCursorSerializer import com.twitter.util.Time import com.twitter.util.Try import javax.inject.Inject import javax.inject.Singleton @Singleton class ListTweetsProductPipelineConfig @Inject() ( listTweetsMixerPipelineConfig: ListTweetsMixerPipelineConfig, listTweetsParamConfig: ListTweetsParamConfig) extends ProductPipelineConfig[HomeMixerRequest, ListTweetsQuery, urt.TimelineResponse] { override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("ListTweets") override val product: request.Product = ListTweetsProduct override val paramConfig: ProductParamConfig = listTweetsParamConfig override val denyLoggedOutUsers: Boolean = false override def pipelineQueryTransformer( request: HomeMixerRequest, params: Params ): ListTweetsQuery = { val context = request.productContext match { case Some(context: ListTweetsProductContext) => context case _ => throw PipelineFailure(BadRequest, "ListTweetsProductContext not found") } val debugOptions = request.debugParams.flatMap(_.debugOptions) /** * Unlike other clients, newly created tweets on Android have the sort index set to the current * time instead of the top sort index + 1, so these tweets get stuck at the top of the timeline * if subsequent timeline responses use the sort index from the previous response instead of * the current time. */ val pipelineCursor = request.serializedRequestCursor.flatMap { cursor => Try(UrtCursorSerializer.deserializeOrderedCursor(cursor)) .getOrElse(ChronologicalCursorUnmarshaller(RequestCursorSerializer.deserialize(cursor))) .map { case UrtOrderedCursor(_, id, Some(GapCursor), gapBoundaryId) if id.isEmpty || gapBoundaryId.isEmpty => throw PipelineFailure(MalformedCursor, "Gap Cursor bounds not defined") case topCursor @ UrtOrderedCursor(_, _, Some(TopCursor), _) => val queryTime = debugOptions.flatMap(_.requestTimeOverride).getOrElse(Time.now) topCursor.copy(initialSortIndex = SortIndexBuilder.timeToId(queryTime)) case cursor => cursor } } ListTweetsQuery( params = params, clientContext = request.clientContext, features = None, pipelineCursor = pipelineCursor, requestedMaxResults = Some(params(ServerMaxResultsParam)), debugOptions = debugOptions, listId = context.listId, deviceContext = context.deviceContext, dspClientContext = context.dspClientContext ) } override def pipelines: Seq[PipelineConfig] = Seq(listTweetsMixerPipelineConfig) override def pipelineSelector(query: ListTweetsQuery): ComponentIdentifier = listTweetsMixerPipelineConfig.identifier override val alerts: Seq[Alert] = Seq( SuccessRateAlert( notificationGroup = DefaultNotificationGroup, warnPredicate = TriggerIfBelow(99.9, 20, 30), criticalPredicate = TriggerIfBelow(99.9, 30, 30), ), LatencyAlert( notificationGroup = DefaultNotificationGroup, percentile = P99, warnPredicate = TriggerIfLatencyAbove(300.millis, 15, 30), criticalPredicate = TriggerIfLatencyAbove(400.millis, 15, 30) ), ThroughputAlert( notificationGroup = DefaultNotificationGroup, warnPredicate = TriggerIfAbove(3000), criticalPredicate = TriggerIfAbove(4000) ), EmptyResponseRateAlert( notificationGroup = DefaultNotificationGroup, warnPredicate = TriggerIfAbove(65), criticalPredicate = TriggerIfAbove(80) ) ) override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsTimelineServiceCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.list_tweets import com.twitter.home_mixer.candidate_pipeline.TimelineServiceResponseFeatureTransformer import com.twitter.home_mixer.marshaller.timelines.TimelineServiceCursorMarshaller import com.twitter.home_mixer.product.list_tweets.model.ListTweetsQuery import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.ServerMaxResultsParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.component_library.candidate_source.timeline_service.TimelineServiceTweetCandidateSource import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelineservice.{thriftscala => t} import javax.inject.Inject import javax.inject.Singleton @Singleton class ListTweetsTimelineServiceCandidatePipelineConfig @Inject() ( timelineServiceTweetCandidateSource: TimelineServiceTweetCandidateSource) extends CandidatePipelineConfig[ListTweetsQuery, t.TimelineQuery, t.Tweet, TweetCandidate] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ListTweetsTimelineServiceTweets") override val queryTransformer: CandidatePipelineQueryTransformer[ ListTweetsQuery, t.TimelineQuery ] = { query => val timelineQueryOptions = t.TimelineQueryOptions( contextualUserId = query.clientContext.userId, ) t.TimelineQuery( timelineType = t.TimelineType.List, timelineId = query.listId, maxCount = query.maxResults(ServerMaxResultsParam).toShort, cursor2 = query.pipelineCursor.flatMap(TimelineServiceCursorMarshaller(_)), options = Some(timelineQueryOptions), timelineId2 = Some(t.TimelineId(t.TimelineType.List, query.listId, None)) ) } override def candidateSource: BaseCandidateSource[t.TimelineQuery, t.Tweet] = timelineServiceTweetCandidateSource override val resultTransformer: CandidatePipelineResultsTransformer[t.Tweet, TweetCandidate] = { sourceResult => TweetCandidate(id = sourceResult.statusId) } override val featuresFromCandidateSourceTransformers: Seq[CandidateFeatureTransformer[t.Tweet]] = Seq(TimelineServiceResponseFeatureTransformer) override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.7) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", "timelines/src/main/scala/com/twitter/timelines/injection/scribe", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/ListConversationServiceCandidateDecorator.scala ================================================ package com.twitter.home_mixer.product.list_tweets.decorator import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder import com.twitter.home_mixer.functional_component.decorator.builder.ListClientEventDetailsBuilder import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.injection.scribe.InjectionScribeUtil import com.twitter.timelineservice.suggests.{thriftscala => st} object ListConversationServiceCandidateDecorator { private val ConversationModuleNamespace = EntryNamespace("list-conversation") def apply(): Some[UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long]] = { val suggestType = st.SuggestType.OrganicListTweet val component = InjectionScribeUtil.scribeComponent(suggestType).get val clientEventInfoBuilder = ClientEventInfoBuilder( component = component, detailsBuilder = Some(ListClientEventDetailsBuilder(st.SuggestType.OrganicListTweet)) ) val tweetItemBuilder = TweetCandidateUrtItemBuilder( clientEventInfoBuilder = clientEventInfoBuilder ) val moduleBuilder = TimelineModuleBuilder( entryNamespace = ConversationModuleNamespace, clientEventInfoBuilder = clientEventInfoBuilder, displayTypeBuilder = StaticModuleDisplayTypeBuilder(VerticalConversation), metadataBuilder = Some(HomeConversationModuleMetadataBuilder()) ) Some( UrtMultipleModulesDecorator( urtItemCandidateDecorator = UrtItemCandidateDecorator(tweetItemBuilder), moduleBuilder = moduleBuilder, groupByKey = (_, _, candidateFeatures) => candidateFeatures.getOrElse(ConversationModuleFocalTweetIdFeature, None) )) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", ], exports = [ "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/ListTweetsQuery.scala ================================================ package com.twitter.home_mixer.product.list_tweets.model import com.twitter.adserver.thriftscala.HomeTimelineType import com.twitter.adserver.thriftscala.TimelineRequestParams import com.twitter.dspbidder.commons.{thriftscala => dsp} import com.twitter.home_mixer.model.HomeAdsQuery import com.twitter.home_mixer.model.request.DeviceContext import com.twitter.home_mixer.model.request.HasListId import com.twitter.home_mixer.model.request.ListTweetsProduct import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.request._ import com.twitter.product_mixer.core.pipeline.HasPipelineCursor import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.configapi.Params case class ListTweetsQuery( override val params: Params, override val clientContext: ClientContext, override val pipelineCursor: Option[UrtOrderedCursor], override val requestedMaxResults: Option[Int], override val debugOptions: Option[DebugOptions], override val features: Option[FeatureMap], override val listId: Long, override val deviceContext: Option[DeviceContext], override val dspClientContext: Option[dsp.DspClientContext]) extends PipelineQuery with HasPipelineCursor[UrtOrderedCursor] with HasListId with HomeAdsQuery { override val product: Product = ListTweetsProduct override def withFeatureMap(features: FeatureMap): ListTweetsQuery = copy(features = Some(features)) override val timelineRequestParams: Option[TimelineRequestParams] = Some(TimelineRequestParams(homeTimelineType = Some(HomeTimelineType.HomeLatest))) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/javax/inject:javax.inject", "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParam.scala ================================================ package com.twitter.home_mixer.product.list_tweets.param import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam object ListTweetsParam { val SupportedClientFSName = "list_tweets_supported_client" object EnableAdsCandidatePipelineParam extends FSParam[Boolean]( name = "list_tweets_enable_ads", default = false ) object ServerMaxResultsParam extends FSBoundedParam[Int]( name = "list_tweets_server_max_results", default = 100, min = 1, max = 500 ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParamConfig.scala ================================================ package com.twitter.home_mixer.product.list_tweets.param import com.twitter.home_mixer.param.decider.DeciderKey import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.EnableAdsCandidatePipelineParam import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.ServerMaxResultsParam import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.SupportedClientFSName import com.twitter.product_mixer.core.product.ProductParamConfig import com.twitter.servo.decider.DeciderKeyName import javax.inject.Inject import javax.inject.Singleton @Singleton class ListTweetsParamConfig @Inject() () extends ProductParamConfig { override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableListTweetsProduct override val supportedClientFSName: String = SupportedClientFSName override val booleanFSOverrides = Seq(EnableAdsCandidatePipelineParam) override val boundedIntFSOverrides = Seq( ServerMaxResultsParam ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer-features/thrift/src/main/thrift:thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/offline_aggregates", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/control_ai", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/location", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/user_fav_avg_embeddings", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation", "src/scala/com/twitter/timelines/prediction/adapters/large_embeddings", "src/thrift/com/twitter/search:earlybird-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets import com.twitter.home_mixer.model.HomeFeatures.ServedAuthorIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SignupCountryFeature import com.twitter.home_mixer.model.HomeFeatures.SignupSourceFeature import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature import com.twitter.home_mixer.model.HomeFeatures.UserFollowersCountFeature import com.twitter.home_mixer.model.HomeFeatures.ViewerAllowsForYouRecommendationsFeature import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.model.request.ScoredTweetsProduct import com.twitter.home_mixer.model.request.ScoredTweetsProductContext import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParamConfig import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.request.Product import com.twitter.product_mixer.core.pipeline.PipelineConfig import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig import com.twitter.product_mixer.core.product.ProductParamConfig import com.twitter.timelines.configapi.Params import javax.inject.Inject import javax.inject.Singleton @Singleton class ScoredTweetsProductPipelineConfig @Inject() ( scoredTweetsRecommendationPipelineConfig: ScoredTweetsRecommendationPipelineConfig, scoredTweetsParamConfig: ScoredTweetsParamConfig) extends ProductPipelineConfig[HomeMixerRequest, ScoredTweetsQuery, t.ScoredTweets] { override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("ScoredTweets") override val product: Product = ScoredTweetsProduct override val paramConfig: ProductParamConfig = scoredTweetsParamConfig override def pipelineQueryTransformer( request: HomeMixerRequest, params: Params ): ScoredTweetsQuery = { val context = request.productContext match { case Some(context: ScoredTweetsProductContext) => context case _ => throw PipelineFailure(BadRequest, "ScoredTweetsProductContext not found") } val featureMap = FeatureMapBuilder() .add(ServedTweetIdsFeature, context.servedTweetIds.getOrElse(Seq.empty)) .add(TimelineServiceTweetsFeature, context.backfillTweetIds.getOrElse(Seq.empty)) .add(SignupCountryFeature, context.signupCountryCode) .add(ViewerAllowsForYouRecommendationsFeature, context.allowForYouRecommendations) .add(SignupSourceFeature, context.signupSource) .add(ServedAuthorIdsFeature, context.servedAuthorIds.getOrElse(Map.empty[Long, Seq[Long]])) .add(UserFollowersCountFeature, context.followerCount) .build() ScoredTweetsQuery( params = params, clientContext = request.clientContext, pipelineCursor = request.serializedRequestCursor.flatMap(UrtCursorSerializer.deserializeOrderedCursor), requestedMaxResults = request.maxResults, debugOptions = request.debugParams.flatMap(_.debugOptions), features = Some(featureMap), deviceContext = context.deviceContext, seenTweetIds = context.seenTweetIds, qualityFactorStatus = None, product = product ) } override val pipelines: Seq[PipelineConfig] = Seq(scoredTweetsRecommendationPipelineConfig) override def pipelineSelector(query: ScoredTweetsQuery): ComponentIdentifier = scoredTweetsRecommendationPipelineConfig.identifier override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.functional_component.feature_hydrator.FollowableUttTopicsQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.CategoryDiversityRescoringFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.DiversityRescoringFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.FeedbackHistoryQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.GizmoduckUserQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.GrokTranslatedPostIsCachedFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.HeartbeatOptimizerParamsHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.HeavyRankerWeightsQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.ImpressedMediaClusterIdsQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.ImpressionBloomFilterQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.ListIdsQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.NaviClientConfigQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.OnPremRealGraphQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.OptimizerWeightsQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.PhoenixRescoringFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.RealGraphInNetworkScoresQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.RealGraphQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.RealTimeEntityRealGraphQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.RealTimeInteractionGraphUserVertexQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.RequestQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.RequestTimeQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.SimClustersUserSparseEmbeddingsQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetTypeMetricsFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TwhinRebuildUserEngagementQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TwhinRebuildUserPositiveQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TwhinUserEngagementQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TwhinUserFollowQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TwhinUserNegativeQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TwhinUserPositiveQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.UnifiedUserActionsUserIdentifierFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.UserActionsQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.UserEngagedGrokCategoriesFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.UserEngagedLanguagesFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.UserHistoryTransformerEmbeddingQueryFeatureHydratorBuilder import com.twitter.home_mixer.functional_component.feature_hydrator.UserLanguagesFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.UserLargeEmbeddingsFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.UserStateQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.UserUnderstandableLanguagesFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.PartAAggregateQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.PartBAggregateQueryFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates.UserEngagementRealTimeAggregatesFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.user_history.ScoredTweetsUserHistoryEventsQueryFeatureHydrator import com.twitter.home_mixer.functional_component.filter.ClipImageClusterDeduplicationFilter import com.twitter.home_mixer.functional_component.filter.ClipVideoClusterDeduplicationFilter import com.twitter.home_mixer.functional_component.filter.FeedbackFatigueFilter import com.twitter.home_mixer.functional_component.filter.GrokGoreFilter import com.twitter.home_mixer.functional_component.filter.GrokNsfwFilter import com.twitter.home_mixer.functional_component.filter.GrokSpamFilter import com.twitter.home_mixer.functional_component.filter.GrokViolentFilter import com.twitter.home_mixer.functional_component.filter.HasAuthorFilter import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter import com.twitter.home_mixer.functional_component.filter.LocationFilter import com.twitter.home_mixer.functional_component.filter.MediaIdDeduplicationFilter import com.twitter.home_mixer.functional_component.filter.MinVideoDurationFilter import com.twitter.home_mixer.functional_component.filter.PreviouslySeenTweetsFilter import com.twitter.home_mixer.functional_component.filter.PreviouslyServedTweetsFilter import com.twitter.home_mixer.functional_component.filter.QuoteDeduplicationFilter import com.twitter.home_mixer.functional_component.filter.RejectTweetFromViewerFilter import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter import com.twitter.home_mixer.functional_component.filter.SlopFilter import com.twitter.home_mixer.functional_component.filter.TweetHydrationFilter import com.twitter.home_mixer.functional_component.selector.SortFixedPositionContentExplorationMixedCandidates import com.twitter.home_mixer.functional_component.selector.SortFixedPositionContentExplorationSimclusterColdPostsCandidates import com.twitter.home_mixer.functional_component.selector.SortFixedPositionDeepRetrievalMixedCandidates import com.twitter.home_mixer.functional_component.side_effect.PublishClientSentImpressionsEventBusSideEffect import com.twitter.home_mixer.functional_component.side_effect.UpdateLastNonPollingTimeSideEffect import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsSupportAccountReplyFeature import com.twitter.home_mixer.model.HomeFeatures.OonNsfwFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableLargeEmbeddingsFeatureHydrationParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableRealGraphQueryFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinRebuildUserEngagementFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinRebuildUserPositiveFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinUserNegativeFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinUserPositiveFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableUserFavAvgTextEmbeddingsQueryFeatureParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableUserHistoryTransformerJointBlueEmbeddingFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.CategoryDiversityRescoringParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.TwhinDiversityRescoringParam import com.twitter.home_mixer.param.HomeMixerFlagName.TargetScoringLatency import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.CachedScoredTweetsCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsBackfillCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsContentExplorationCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsDirectUtegCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsListsCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsStaticCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsTweetMixerCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird.ScoredTweetsCommunitiesCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird.ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.CachedScoredTweetsQueryFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.InvalidateCachedScoredTweetsQueryFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.IsColdStartPostFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.LowSignalUserQueryFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.SGSMutuallyFollowedUserHydrator import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieVisibilityFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ValidLikedByUserIdsFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.filter.ControlAiExcludeFilter import com.twitter.home_mixer.product.scored_tweets.filter.ControlAiOnlyIncludeFilter import com.twitter.home_mixer.product.scored_tweets.filter.CustomSnowflakeIdAgeFilter import com.twitter.home_mixer.product.scored_tweets.filter.DuplicateConversationTweetsFilter import com.twitter.home_mixer.product.scored_tweets.filter.GrokAutoTranslateLanguageFilter import com.twitter.home_mixer.product.scored_tweets.filter.IsOutOfNetworkColdStartPostFilter import com.twitter.home_mixer.product.scored_tweets.filter.LanguageFilter import com.twitter.home_mixer.product.scored_tweets.filter.SGSAuthorFilter import com.twitter.home_mixer.product.scored_tweets.marshaller.ScoredTweetsResponseDomainMarshaller import com.twitter.home_mixer.product.scored_tweets.marshaller.ScoredTweetsResponseTransportMarshaller import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.DefaultRequestedMaxResultsParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableClipImageClusterDedupingParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableColdStartFilterParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableContentExplorationSimclusterColdPostsCandidateBoostingParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableContentExplorationCandidatePipelineParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableContentExplorationMixedCandidateBoostingParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableControlAiParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableDeepRetrievalMixedCandidateBoostingParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableHeartbeatOptimizerWeightsParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableMediaClusterDedupingParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableMediaDedupingParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnablePhoenixRescoreParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableRecentEngagementCacheRefreshParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ServerMaxResultsParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableMediaClusterDecayParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableRealTimeEntityRealGraphFeaturesParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableTopicSocialProofFeaturesParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableUserActionsFeatureParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableUserEngagedLanguagesFeaturesParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableUserHistoryEventsFeaturesParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableUserIdentifierFeaturesParam import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsHeuristicScoringPipelineConfig import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsLowSignalScoringPipelineConfig import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsModelScoringPipelineConfig import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsRerankingScoringPipelineConfig import com.twitter.home_mixer.product.scored_tweets.selector.KeepTopKCandidatesPerCommunity import com.twitter.home_mixer.product.scored_tweets.side_effect.CacheCandidateFeaturesSideEffect import com.twitter.home_mixer.product.scored_tweets.side_effect.CacheRequestInfoSideEffect import com.twitter.home_mixer.product.scored_tweets.side_effect.CacheRetrievalSignalSideEffect import com.twitter.home_mixer.product.scored_tweets.side_effect.CachedScoredTweetsSideEffect import com.twitter.home_mixer.product.scored_tweets.side_effect.CommonFeaturesSideEffect import com.twitter.home_mixer.product.scored_tweets.side_effect.ScoredCandidateFeatureKeysKafkaSideEffect import com.twitter.home_mixer.product.scored_tweets.side_effect.ScoredContentExplorationCandidateScoreFeatureKafkaSideEffect import com.twitter.home_mixer.product.scored_tweets.side_effect.ScoredPhoenixCandidatesKafkaSideEffect import com.twitter.home_mixer.product.scored_tweets.side_effect.ScoredStatsSideEffect import com.twitter.home_mixer.product.scored_tweets.side_effect.ScoredTweetsDiversityStatsSideEffect import com.twitter.home_mixer.product.scored_tweets.side_effect.ScribeScoredCandidatesSideEffect import com.twitter.home_mixer.{thriftscala => t} import com.twitter.inject.annotations.Flag import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedBulkCandidateFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.communities.CommunityMembershipsQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.control_ai.ControlAiQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.location.UserLocationQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.AsyncParamGatedQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.user_fav_avg_embeddings.UserFavAvgTextEmbeddingsQueryFeatureHydrator import com.twitter.product_mixer.component_library.filter.ParamGatedFilter import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.selector.DropDuplicateCandidates import com.twitter.product_mixer.component_library.selector.DropRequestedMaxResults import com.twitter.product_mixer.component_library.selector.IdAndClassDuplicationKey import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.component_library.selector.PickFirstCandidateMerger import com.twitter.product_mixer.component_library.selector.SelectConditionally import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates import com.twitter.product_mixer.component_library.selector.sorter.FeatureValueSorter import com.twitter.product_mixer.core.functional_component.common.AllPipelines import com.twitter.product_mixer.core.functional_component.configapi.StaticParam import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier import com.twitter.product_mixer.core.pipeline.FailOpenPolicy import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineConfig import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig import com.twitter.product_mixer.core.quality_factor.BoundsWithDefault import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorConfig import com.twitter.product_mixer.core.quality_factor.QualityFactorConfig import com.twitter.util.Duration import javax.inject.Inject import javax.inject.Singleton @Singleton class ScoredTweetsRecommendationPipelineConfig @Inject() ( // Candidate pipelines scoredTweetsTweetMixerCandidatePipelineConfig: ScoredTweetsTweetMixerCandidatePipelineConfig, scoredTweetsStaticCandidatePipelineConfig: ScoredTweetsStaticCandidatePipelineConfig, scoredTweetsListsCandidatePipelineConfig: ScoredTweetsListsCandidatePipelineConfig, scoredTweetsBackfillCandidatePipelineConfig: ScoredTweetsBackfillCandidatePipelineConfig, scoredTweetsEarlybirdInNetworkCandidatePipelineConfig: ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig, scoredTweetsCommunitiesCandidatePipelineConfig: ScoredTweetsCommunitiesCandidatePipelineConfig, scoredTweetsDirectUtegCandidatePipelineConfig: ScoredTweetsDirectUtegCandidatePipelineConfig, scoredTweetsContentExplorationCandidatePipelineConfig: ScoredTweetsContentExplorationCandidatePipelineConfig, cachedScoredTweetsCandidatePipelineConfig: CachedScoredTweetsCandidatePipelineConfig, // Query feature hydrators followableUttTopicsQueryFeatureHydrator: FollowableUttTopicsQueryFeatureHydrator, gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, controlAiQueryFeatureHydrator: ControlAiQueryFeatureHydrator, lowSignalUserQueryFeatureHydrator: LowSignalUserQueryFeatureHydrator, naviClientConfigQueryFeatureHydrator: NaviClientConfigQueryFeatureHydrator, requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ScoredTweetsQuery], requestTimeQueryFeatureHydrator: RequestTimeQueryFeatureHydrator, realTimeInteractionGraphUserVertexQueryFeatureHydrator: RealTimeInteractionGraphUserVertexQueryFeatureHydrator, sgsMutuallyFollowedUserHydrator: SGSMutuallyFollowedUserHydrator, simClustersUserSparseEmbeddingsQueryFeatureHydrator: SimClustersUserSparseEmbeddingsQueryFeatureHydrator, userLocationQueryFeatureHydrator: UserLocationQueryFeatureHydrator, userStateQueryFeatureHydrator: UserStateQueryFeatureHydrator, userEngagementRealTimeAggregatesFeatureHydrator: UserEngagementRealTimeAggregatesFeatureHydrator, twhinRebuildUserEngagementQueryFeatureHydrator: TwhinRebuildUserEngagementQueryFeatureHydrator, twhinRebuildUserPositiveQueryFeatureHydrator: TwhinRebuildUserPositiveQueryFeatureHydrator, twhinUserEngagementQueryFeatureHydrator: TwhinUserEngagementQueryFeatureHydrator, twhinUserFollowQueryFeatureHydrator: TwhinUserFollowQueryFeatureHydrator, twhinUserPositiveQueryFeatureHydrator: TwhinUserPositiveQueryFeatureHydrator, twhinUserNegativeQueryFeatureHydrator: TwhinUserNegativeQueryFeatureHydrator, userHistoryTransformerEmbeddingQueryFeatureHydratorBuilder: UserHistoryTransformerEmbeddingQueryFeatureHydratorBuilder, cachedScoredTweetsQueryFeatureHydrator: CachedScoredTweetsQueryFeatureHydrator, sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, scoredTweetsModelScoringPipelineConfig: ScoredTweetsModelScoringPipelineConfig, scoredTweetsRerankingScoringPipelineConfig: ScoredTweetsRerankingScoringPipelineConfig, impressionBloomFilterQueryFeatureHydrator: ImpressionBloomFilterQueryFeatureHydrator, memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, userEngagementsAvgTextEmbeddingsQueryFeatureHydrator: UserFavAvgTextEmbeddingsQueryFeatureHydrator, listIdsQueryFeatureHydrator: ListIdsQueryFeatureHydrator, feedbackHistoryQueryFeatureHydrator: FeedbackHistoryQueryFeatureHydrator, communityMembershipsQueryFeatureHydrator: CommunityMembershipsQueryFeatureHydrator, tweetypieVisibilityFeatureHydrator: TweetypieVisibilityFeatureHydrator, realGraphInNetworkScoresQueryFeatureHydrator: RealGraphInNetworkScoresQueryFeatureHydrator, realGraphQueryFeatureHydrator: RealGraphQueryFeatureHydrator, onPremRealGraphQueryFeatureHydrator: OnPremRealGraphQueryFeatureHydrator, entityRealGraphQueryFeatureHydrator: RealTimeEntityRealGraphQueryFeatureHydrator, unifiedUserActionsUserIdentifierFeatureHydrator: UnifiedUserActionsUserIdentifierFeatureHydrator, userEngagedLanguagesFeatureHydrator: UserEngagedLanguagesFeatureHydrator, userLanguagesFeatureHydrator: UserLanguagesFeatureHydrator, userUnderstandableLanguagesFeatureHydrator: UserUnderstandableLanguagesFeatureHydrator, partAAggregateQueryFeatureHydrator: PartAAggregateQueryFeatureHydrator, partBAggregateQueryFeatureHydrator: PartBAggregateQueryFeatureHydrator, heavyRankerWeightsQueryFeatureHydrator: HeavyRankerWeightsQueryFeatureHydrator, userLargeEmbeddingsFeatureHydrator: UserLargeEmbeddingsFeatureHydrator, userHistoryEventsQueryFeatureHydrator: ScoredTweetsUserHistoryEventsQueryFeatureHydrator, userActionsQueryFeatureHydrator: UserActionsQueryFeatureHydrator, impressedMediaClusterIdsQueryFeatureHydrator: ImpressedMediaClusterIdsQueryFeatureHydrator, heartbeatOptimizerParamsHydrator: HeartbeatOptimizerParamsHydrator, optimizerWeightsQueryFeatureHydrator: OptimizerWeightsQueryFeatureHydrator, userEngagedGrokCategoriesFeatureHydrator: UserEngagedGrokCategoriesFeatureHydrator, grokTranslatedPostIsCachedFeatureHydrator: GrokTranslatedPostIsCachedFeatureHydrator, isColdStartPostFeatureHydrator: IsColdStartPostFeatureHydrator, // Filters invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, sgsAuthorFilter: SGSAuthorFilter, // Side effects cacheCandidateFeaturesSideEffect: CacheCandidateFeaturesSideEffect, cachedScoredTweetsSideEffect: CachedScoredTweetsSideEffect, cacheRetrievalSignalSideEffect: CacheRetrievalSignalSideEffect, cacheRequestInfoSideEffect: CacheRequestInfoSideEffect, scoredCandidateFeatureKeysKafkaSideEffect: ScoredCandidateFeatureKeysKafkaSideEffect, scoredContentExplorationCandidateScoreFeatureKafkaSideEffect: ScoredContentExplorationCandidateScoreFeatureKafkaSideEffect, publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, scoredStatsSideEffect: ScoredStatsSideEffect, scoredTweetsDiversityStatsSideEffect: ScoredTweetsDiversityStatsSideEffect, scoredPhoenixCandidatesKafkaSideEffect: ScoredPhoenixCandidatesKafkaSideEffect, scribeCommonFeaturesSideEffect: CommonFeaturesSideEffect, scribeScoredCandidatesSideEffect: ScribeScoredCandidatesSideEffect, updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[ ScoredTweetsQuery, ScoredTweetsResponse ], @Flag(TargetScoringLatency) targetScoringLatency: Duration) extends RecommendationPipelineConfig[ ScoredTweetsQuery, TweetCandidate, ScoredTweetsResponse, t.ScoredTweetsResponse ] { override val identifier: RecommendationPipelineIdentifier = RecommendationPipelineIdentifier("ScoredTweets") private val SubscriptionReplyFilterId = "SubscriptionReply" private val OutOfNetworkNSFWFilterId = "OutOfNetworkNSFW" private val scoringStep = RecommendationPipelineConfig.scoringPipelinesStep override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ScoredTweetsQuery]] = Seq( naviClientConfigQueryFeatureHydrator, requestQueryFeatureHydrator, realGraphInNetworkScoresQueryFeatureHydrator, cachedScoredTweetsQueryFeatureHydrator, sgsFollowedUsersQueryFeatureHydrator, memcacheTweetImpressionsQueryFeatureHydrator, listIdsQueryFeatureHydrator, communityMembershipsQueryFeatureHydrator, userStateQueryFeatureHydrator, lowSignalUserQueryFeatureHydrator, feedbackHistoryQueryFeatureHydrator, requestTimeQueryFeatureHydrator, userUnderstandableLanguagesFeatureHydrator, ParamGatedQueryFeatureHydrator( EnableContentExplorationCandidatePipelineParam, userEngagedGrokCategoriesFeatureHydrator ), AsyncQueryFeatureHydrator( RecommendationPipelineConfig.globalFiltersStep, userLocationQueryFeatureHydrator ), AsyncQueryFeatureHydrator( RecommendationPipelineConfig.globalFiltersStep, impressionBloomFilterQueryFeatureHydrator ), AsyncParamGatedQueryFeatureHydrator( EnableRealGraphQueryFeaturesParam, scoringStep, realGraphQueryFeatureHydrator ), AsyncParamGatedQueryFeatureHydrator( EnableRealGraphQueryFeaturesParam, scoringStep, onPremRealGraphQueryFeatureHydrator ), AsyncParamGatedQueryFeatureHydrator( EnableRealTimeEntityRealGraphFeaturesParam, scoringStep, entityRealGraphQueryFeatureHydrator ), AsyncParamGatedQueryFeatureHydrator( EnableTwhinRebuildUserEngagementFeaturesParam, scoringStep, twhinRebuildUserEngagementQueryFeatureHydrator ), AsyncParamGatedQueryFeatureHydrator( EnableTwhinRebuildUserPositiveFeaturesParam, scoringStep, twhinRebuildUserPositiveQueryFeatureHydrator ), AsyncParamGatedQueryFeatureHydrator( EnableTwhinUserNegativeFeaturesParam, scoringStep, twhinUserNegativeQueryFeatureHydrator ), AsyncParamGatedQueryFeatureHydrator( EnableTwhinUserPositiveFeaturesParam, scoringStep, twhinUserPositiveQueryFeatureHydrator ), AsyncQueryFeatureHydrator( scoringStep, userHistoryTransformerEmbeddingQueryFeatureHydratorBuilder.buildHomeBlueHydrator() ), AsyncQueryFeatureHydrator( scoringStep, userHistoryTransformerEmbeddingQueryFeatureHydratorBuilder.buildHomeGreenHydrator() ), AsyncParamGatedQueryFeatureHydrator( EnableUserHistoryTransformerJointBlueEmbeddingFeaturesParam, scoringStep, userHistoryTransformerEmbeddingQueryFeatureHydratorBuilder.buildJointBlueHydrator() ), AsyncParamGatedQueryFeatureHydrator( EnableTopicSocialProofFeaturesParam, scoringStep, followableUttTopicsQueryFeatureHydrator ), AsyncParamGatedQueryFeatureHydrator( EnableUserEngagedLanguagesFeaturesParam, scoringStep, userEngagedLanguagesFeatureHydrator ), AsyncParamGatedQueryFeatureHydrator( EnableLargeEmbeddingsFeatureHydrationParam, scoringStep, userLargeEmbeddingsFeatureHydrator, ), AsyncParamGatedQueryFeatureHydrator( EnableUserHistoryEventsFeaturesParam, scoringStep, userHistoryEventsQueryFeatureHydrator, ), AsyncParamGatedQueryFeatureHydrator( EnableUserActionsFeatureParam, scoringStep, userActionsQueryFeatureHydrator ), AsyncParamGatedQueryFeatureHydrator( EnableMediaClusterDecayParam, scoringStep, impressedMediaClusterIdsQueryFeatureHydrator, ), AsyncParamGatedQueryFeatureHydrator( EnableUserIdentifierFeaturesParam, scoringStep, unifiedUserActionsUserIdentifierFeatureHydrator ), AsyncQueryFeatureHydrator(scoringStep, userLanguagesFeatureHydrator), AsyncQueryFeatureHydrator(scoringStep, sgsMutuallyFollowedUserHydrator), AsyncQueryFeatureHydrator(scoringStep, userEngagementRealTimeAggregatesFeatureHydrator), AsyncQueryFeatureHydrator(scoringStep, realTimeInteractionGraphUserVertexQueryFeatureHydrator), AsyncQueryFeatureHydrator(scoringStep, twhinUserFollowQueryFeatureHydrator), AsyncQueryFeatureHydrator(scoringStep, twhinUserEngagementQueryFeatureHydrator), AsyncQueryFeatureHydrator(scoringStep, partAAggregateQueryFeatureHydrator), AsyncQueryFeatureHydrator(scoringStep, partBAggregateQueryFeatureHydrator), AsyncQueryFeatureHydrator(scoringStep, heavyRankerWeightsQueryFeatureHydrator), ParamGatedQueryFeatureHydrator( EnableHeartbeatOptimizerWeightsParam, heartbeatOptimizerParamsHydrator ), AsyncQueryFeatureHydrator(scoringStep, simClustersUserSparseEmbeddingsQueryFeatureHydrator), AsyncParamGatedQueryFeatureHydrator( EnableControlAiParam, scoringStep, controlAiQueryFeatureHydrator ), AsyncParamGatedQueryFeatureHydrator( EnableUserFavAvgTextEmbeddingsQueryFeatureParam, RecommendationPipelineConfig.resultSideEffectsStep, userEngagementsAvgTextEmbeddingsQueryFeatureHydrator ), ) override val fetchQueryFeaturesPhase2: Seq[QueryFeatureHydrator[ScoredTweetsQuery]] = Seq( AsyncParamGatedQueryFeatureHydrator( EnableHeartbeatOptimizerWeightsParam, scoringStep, optimizerWeightsQueryFeatureHydrator ), ParamGatedQueryFeatureHydrator( EnableRecentEngagementCacheRefreshParam, InvalidateCachedScoredTweetsQueryFeatureHydrator ) ) override val candidatePipelines: Seq[ CandidatePipelineConfig[ScoredTweetsQuery, _, _, TweetCandidate] ] = Seq( // Order matters. Duplicated candidates take the first occurrence in the pipelines when merger // strategy is PickFirstCandidateMerger. scoredTweetsStaticCandidatePipelineConfig, cachedScoredTweetsCandidatePipelineConfig, scoredTweetsEarlybirdInNetworkCandidatePipelineConfig, scoredTweetsDirectUtegCandidatePipelineConfig, scoredTweetsTweetMixerCandidatePipelineConfig, scoredTweetsContentExplorationCandidatePipelineConfig, scoredTweetsListsCandidatePipelineConfig, scoredTweetsBackfillCandidatePipelineConfig, scoredTweetsCommunitiesCandidatePipelineConfig ) override val postCandidatePipelinesSelectors: Seq[Selector[ScoredTweetsQuery]] = Seq( DropDuplicateCandidates( pipelineScope = AllPipelines, duplicationKey = IdAndClassDuplicationKey, mergeStrategy = PickFirstCandidateMerger ), InsertAppendResults(AllPipelines) ) override val globalFilters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( // sort these to have the "cheaper" filters run first CustomSnowflakeIdAgeFilter(StaticParam(48.hours)), HasAuthorFilter, RejectTweetFromViewerFilter, RetweetDeduplicationFilter, LocationFilter, PreviouslySeenTweetsFilter, PreviouslyServedTweetsFilter, PredicateFeatureFilter.fromPredicate( FilterIdentifier(SubscriptionReplyFilterId), shouldKeepCandidate = { features => features.getOrElse(InReplyToTweetIdFeature, None).isEmpty || features.getOrElse(ExclusiveConversationAuthorIdFeature, None).isEmpty } ), FeedbackFatigueFilter, invalidSubscriptionTweetFilter, sgsAuthorFilter ) override val candidatePipelineFailOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( scoredTweetsStaticCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, cachedScoredTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, scoredTweetsTweetMixerCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, scoredTweetsListsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, scoredTweetsBackfillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, scoredTweetsEarlybirdInNetworkCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, scoredTweetsCommunitiesCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, scoredTweetsDirectUtegCandidatePipelineConfig.identifier -> FailOpenPolicy.Always ) override val scoringPipelineFailOpenPolicies: Map[ScoringPipelineIdentifier, FailOpenPolicy] = Map( ScoredTweetsHeuristicScoringPipelineConfig.identifier -> FailOpenPolicy.Always, ScoredTweetsLowSignalScoringPipelineConfig.identifier -> FailOpenPolicy.Always ) private val scoringPipelineQualityFactorConfig = LinearLatencyQualityFactorConfig( qualityFactorBounds = BoundsWithDefault(minInclusive = 0.1, maxInclusive = 1.0, default = 0.95), initialDelay = 60.seconds, targetLatency = targetScoringLatency, targetLatencyPercentile = 95.0, delta = 0.00125 ) override val qualityFactorConfigs: Map[ComponentIdentifier, QualityFactorConfig] = Map( scoredTweetsModelScoringPipelineConfig.identifier -> scoringPipelineQualityFactorConfig, ) override val scoringPipelines: Seq[ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate]] = Seq( // scoring pipeline - run on non-cached candidates only since cached ones are already scored scoredTweetsModelScoringPipelineConfig, scoredTweetsRerankingScoringPipelineConfig, // re-scoring pipeline - run on all candidates since these are request specific ScoredTweetsHeuristicScoringPipelineConfig, ScoredTweetsLowSignalScoringPipelineConfig, ) override val postScoringFilters = Seq( MinVideoDurationFilter, ParamGatedFilter( EnableControlAiParam, ControlAiExcludeFilter ), ParamGatedFilter( EnableControlAiParam, ControlAiOnlyIncludeFilter ), SlopFilter, GrokGoreFilter, GrokNsfwFilter, GrokSpamFilter, GrokViolentFilter, LanguageFilter, GrokAutoTranslateLanguageFilter, DuplicateConversationTweetsFilter, PredicateFeatureFilter.fromPredicate( FilterIdentifier("IsSupportAccountReply"), shouldKeepCandidate = { features => !features.getOrElse(IsSupportAccountReplyFeature, false) } ), ) override val resultSelectors: Seq[Selector[ScoredTweetsQuery]] = Seq( KeepTopKCandidatesPerCommunity(AllPipelines), UpdateSortCandidates(AllPipelines, FeatureValueSorter.descending(ScoreFeature)), SelectConditionally.paramGated( SortFixedPositionContentExplorationMixedCandidates, EnableContentExplorationMixedCandidateBoostingParam ), SelectConditionally.paramGated( SortFixedPositionDeepRetrievalMixedCandidates, EnableDeepRetrievalMixedCandidateBoostingParam ), SelectConditionally.paramGated( SortFixedPositionContentExplorationSimclusterColdPostsCandidates, EnableContentExplorationSimclusterColdPostsCandidateBoostingParam ), InsertAppendResults(AllPipelines), DropRequestedMaxResults( defaultRequestedMaxResultsParam = DefaultRequestedMaxResultsParam, serverMaxResultsParam = ServerMaxResultsParam ) ) override val postSelectionFeatureHydration = Seq( grokTranslatedPostIsCachedFeatureHydrator, tweetypieVisibilityFeatureHydrator, TweetTypeMetricsFeatureHydrator, ParamGatedBulkCandidateFeatureHydrator( EnablePhoenixRescoreParam, PhoenixRescoringFeatureHydrator ), ParamGatedBulkCandidateFeatureHydrator( TwhinDiversityRescoringParam, DiversityRescoringFeatureHydrator ), ParamGatedBulkCandidateFeatureHydrator( CategoryDiversityRescoringParam, CategoryDiversityRescoringFeatureHydrator ), ParamGatedCandidateFeatureHydrator( EnableColdStartFilterParam, isColdStartPostFeatureHydrator ), ValidLikedByUserIdsFeatureHydrator ) override val postSelectionFilters = Seq( TweetHydrationFilter, PredicateFeatureFilter.fromPredicate( FilterIdentifier(OutOfNetworkNSFWFilterId), shouldKeepCandidate = { features => !features.getOrElse(OonNsfwFeature, false) } ), QuoteDeduplicationFilter, ParamGatedFilter( EnableMediaDedupingParam, MediaIdDeduplicationFilter ), ParamGatedFilter( EnableMediaClusterDedupingParam, ClipVideoClusterDeduplicationFilter ), ParamGatedFilter( EnableClipImageClusterDedupingParam, ClipImageClusterDeduplicationFilter ), ParamGatedFilter( EnableColdStartFilterParam, IsOutOfNetworkColdStartPostFilter ) ) override val resultSideEffects: Seq[ PipelineResultSideEffect[ScoredTweetsQuery, ScoredTweetsResponse] ] = Seq( cacheCandidateFeaturesSideEffect, cachedScoredTweetsSideEffect, scoredCandidateFeatureKeysKafkaSideEffect, scoredContentExplorationCandidateScoreFeatureKafkaSideEffect, scoredPhoenixCandidatesKafkaSideEffect, publishClientSentImpressionsEventBusSideEffect, scoredStatsSideEffect, scoredTweetsDiversityStatsSideEffect, scribeCommonFeaturesSideEffect, scribeScoredCandidatesSideEffect, updateLastNonPollingTimeSideEffect, cacheRetrievalSignalSideEffect, cacheRequestInfoSideEffect ) override val domainMarshaller: DomainMarshaller[ ScoredTweetsQuery, ScoredTweetsResponse ] = ScoredTweetsResponseDomainMarshaller override val transportMarshaller: TransportMarshaller[ ScoredTweetsResponse, t.ScoredTweetsResponse ] = ScoredTweetsResponseTransportMarshaller } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweet_mixer", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/uteg", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/module:test-user-mapper", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", "src/thrift/com/twitter/timelineranker:thrift-scala", "tweet-mixer/thrift/src/main/thrift:thrift-scala", ], exports = [ "src/thrift/com/twitter/timelineranker:thrift-scala", "tweet-mixer/thrift/src/main/thrift:thrift-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.CachedScoredTweetsCandidatePipelineConfig._ import com.twitter.home_mixer.product.scored_tweets.candidate_source.CachedScoredTweetsCandidateSource import com.twitter.home_mixer.product.scored_tweets.gate.RecentFeedbackCheckGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.response_transformer.CachedScoredTweetsResponseFeatureTransformer import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import javax.inject.Inject import javax.inject.Singleton /** * Candidate Pipeline Config that fetches tweets from Scored Tweets Cache. */ @Singleton class CachedScoredTweetsCandidatePipelineConfig @Inject() ( cachedScoredTweetsCandidateSource: CachedScoredTweetsCandidateSource) extends CandidatePipelineConfig[ ScoredTweetsQuery, ScoredTweetsQuery, hmt.ScoredTweet, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = Identifier override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( NonEmptySeqFeatureGate(CachedScoredTweetsFeature), RecentFeedbackCheckGate ) override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, ScoredTweetsQuery ] = identity override val candidateSource: BaseCandidateSource[ScoredTweetsQuery, hmt.ScoredTweet] = cachedScoredTweetsCandidateSource override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[hmt.ScoredTweet] ] = Seq(CachedScoredTweetsResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ hmt.ScoredTweet, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } } object CachedScoredTweetsCandidatePipelineConfig { val Identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("CachedScoredTweets") } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline import com.twitter.home_mixer.functional_component.feature_hydrator.TweetEntityServiceFeatureHydrator import com.twitter.home_mixer.functional_component.filter.HasAuthorFilter import com.twitter.home_mixer.functional_component.filter.ReplyFilter import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature import com.twitter.home_mixer.product.scored_tweets.gate.DenyLowSignalUserGate import com.twitter.home_mixer.product.scored_tweets.gate.MinTimeSinceLastRequestGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableBackfillCandidatePipelineParam import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsBackfillResponseFeatureTransformer import com.twitter.product_mixer.component_library.gate.EmptySeqFeatureGate import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.candidate_source.PassthroughCandidateSource import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @Singleton class ScoredTweetsBackfillCandidatePipelineConfig @Inject() ( tweetEntityServiceFeatureHydrator: TweetEntityServiceFeatureHydrator) extends CandidatePipelineConfig[ ScoredTweetsQuery, ScoredTweetsQuery, Long, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ScoredTweetsBackfill") override val supportedClientParam: Option[FSParam[Boolean]] = Some(EnableBackfillCandidatePipelineParam) override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( MinTimeSinceLastRequestGate, DenyLowSignalUserGate, NonEmptySeqFeatureGate(TimelineServiceTweetsFeature), EmptySeqFeatureGate(CachedScoredTweetsFeature) ) override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, ScoredTweetsQuery ] = identity override def candidateSource: CandidateSource[ScoredTweetsQuery, Long] = PassthroughCandidateSource( identifier = CandidateSourceIdentifier("ScoredTweetsBackfill"), candidateExtractor = { query => query.features.map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty)).toSeq.flatten } ) override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[Long] ] = Seq(ScoredTweetsBackfillResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[Long, TweetCandidate] = { sourceResult => TweetCandidate(id = sourceResult) } override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] ] = Seq(tweetEntityServiceFeatureHydrator) override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( ReplyFilter, HasAuthorFilter ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsContentExplorationCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline import com.twitter.home_mixer.functional_component.feature_hydrator.TweetEntityServiceFeatureHydrator import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature import com.twitter.home_mixer.product.scored_tweets.candidate_source.ContentExplorationCandidateResponse import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.candidate_source.ContentExplorationCandidateSource import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableContentExplorationCandidatePipelineParam import com.twitter.home_mixer.product.scored_tweets.query_transformer.ContentExplorationQueryRequest import com.twitter.home_mixer.product.scored_tweets.query_transformer.ContentExplorationQueryTransformer import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsContentExplorationResponseFeatureTransformer import com.twitter.product_mixer.component_library.gate.EmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton /** * Candidate Pipeline Config that fetches tweets from the content exploration post source */ @Singleton class ScoredTweetsContentExplorationCandidatePipelineConfig @Inject() ( contentExplorationCandidateSource: ContentExplorationCandidateSource, tweetEntityServiceFeatureHydrator: TweetEntityServiceFeatureHydrator) extends CandidatePipelineConfig[ ScoredTweetsQuery, ContentExplorationQueryRequest, ContentExplorationCandidateResponse, TweetCandidate ] { override val supportedClientParam: Option[FSParam[Boolean]] = Some( EnableContentExplorationCandidatePipelineParam) override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( EmptySeqFeatureGate(CachedScoredTweetsFeature)) override val identifier: CandidatePipelineIdentifier = ScoredTweetsContentExplorationCandidatePipelineConfig.identifier override val candidateSource: ContentExplorationCandidateSource = contentExplorationCandidateSource override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, ContentExplorationQueryRequest ] = ContentExplorationQueryTransformer override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[ContentExplorationCandidateResponse] ] = Seq(ScoredTweetsContentExplorationResponseFeatureTransformer) override val postFilterFeatureHydration: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] ] = Seq(tweetEntityServiceFeatureHydrator) override val resultTransformer: CandidatePipelineResultsTransformer[ ContentExplorationCandidateResponse, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } } object ScoredTweetsContentExplorationCandidatePipelineConfig { val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ScoredTweetsContentExploration") } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsDirectUtegCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline import com.twitter.home_mixer.functional_component.feature_hydrator.EarlybirdSearchResultFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetEntityServiceFeatureHydrator import com.twitter.home_mixer.functional_component.gate.AllowForYouRecommendationsGate import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature import com.twitter.home_mixer.product.scored_tweets.filter.FollowedAuthorFilter import com.twitter.home_mixer.product.scored_tweets.filter.OONReplyFilter import com.twitter.home_mixer.product.scored_tweets.filter.UtegMinFavCountFilter import com.twitter.home_mixer.product.scored_tweets.filter.UtegTopKFilter import com.twitter.home_mixer.product.scored_tweets.gate.DenyLowSignalUserGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidateSourceParams import com.twitter.home_mixer.product.scored_tweets.query_transformer.UtegQueryTransformer import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsDirectUtegResponseFeatureTransformer import com.twitter.product_mixer.component_library.candidate_source.uteg.UtegCandidateSource import com.twitter.product_mixer.component_library.gate.EmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @Singleton class ScoredTweetsDirectUtegCandidatePipelineConfig @Inject() ( utegCandidateSource: UtegCandidateSource, tweetEntityServiceFeatureHydrator: TweetEntityServiceFeatureHydrator, earlybirdSearchResultFeatureHydrator: EarlybirdSearchResultFeatureHydrator) extends CandidatePipelineConfig[ ScoredTweetsQuery, uteg.RecommendTweetEntityRequest, uteg.TweetRecommendation, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ScoredTweetsDirectUteg") override val supportedClientParam: Option[FSParam[Boolean]] = Some( CandidateSourceParams.EnableUTEGCandidateSourceParam) override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( AllowForYouRecommendationsGate, DenyLowSignalUserGate, EmptySeqFeatureGate(CachedScoredTweetsFeature) ) override val candidateSource: BaseCandidateSource[ uteg.RecommendTweetEntityRequest, uteg.TweetRecommendation ] = utegCandidateSource override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, uteg.RecommendTweetEntityRequest ] = UtegQueryTransformer(identifier) override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] ] = Seq( tweetEntityServiceFeatureHydrator, earlybirdSearchResultFeatureHydrator ) override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[uteg.TweetRecommendation] ] = Seq(ScoredTweetsDirectUtegResponseFeatureTransformer) override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( FollowedAuthorFilter, UtegMinFavCountFilter, OONReplyFilter, UtegTopKFilter, ) override val resultTransformer: CandidatePipelineResultsTransformer[ uteg.TweetRecommendation, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsFrsCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FrsSeedUsersQueryFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerFrsQueryTransformer import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsFrsResponseFeatureTransformer import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerRecapCandidateSource import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelineranker.{thriftscala => tlr} import com.twitter.timelines.configapi.decider.DeciderParam import javax.inject.Inject import javax.inject.Singleton /** * Candidate Pipeline Config that takes user recommendations from Follow Recommendation Service (FRS) * and makes a TimelineRanker->Earlybird query for tweet candidates from those users. * Additionally, the candidate pipeline hydrates followedByUserIds so that followed-by social proof * can be used. */ @Singleton class ScoredTweetsFrsCandidatePipelineConfig @Inject() ( timelineRankerRecapCandidateSource: TimelineRankerRecapCandidateSource, frsSeedUsersQueryFeatureHydrator: FrsSeedUsersQueryFeatureHydrator) extends CandidatePipelineConfig[ ScoredTweetsQuery, tlr.RecapQuery, tlr.CandidateTweet, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ScoredTweetsFrs") override val enabledDeciderParam: Option[DeciderParam[Boolean]] = Some(CandidatePipeline.EnableFrsParam) override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) ) override val queryFeatureHydration: Seq[ BaseQueryFeatureHydrator[ScoredTweetsQuery, _] ] = Seq(frsSeedUsersQueryFeatureHydrator) override val candidateSource: BaseCandidateSource[tlr.RecapQuery, tlr.CandidateTweet] = timelineRankerRecapCandidateSource override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, tlr.RecapQuery ] = TimelineRankerFrsQueryTransformer(identifier) override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[tlr.CandidateTweet] ] = Seq(ScoredTweetsFrsResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ tlr.CandidateTweet, TweetCandidate ] = { candidate => TweetCandidate(candidate.tweet.get.id) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsInNetworkCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.IsExtendedReplyFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ReplyFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RetweetSourceTweetFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.filter.RetweetSourceTweetRemovingFilter import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerInNetworkQueryTransformer import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsInNetworkResponseFeatureTransformer import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerInNetworkCandidateSource import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelineranker.{thriftscala => t} import com.twitter.timelines.configapi.decider.DeciderParam import javax.inject.Inject import javax.inject.Singleton /** * Candidate Pipeline Config to fetch in-network tweets from Timeline Ranker's Recycled source */ @Singleton class ScoredTweetsInNetworkCandidatePipelineConfig @Inject() ( timelineRankerInNetworkCandidateSource: TimelineRankerInNetworkCandidateSource, replyFeatureHydrator: ReplyFeatureHydrator) extends CandidatePipelineConfig[ ScoredTweetsQuery, t.RecapQuery, t.CandidateTweet, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ScoredTweetsInNetwork") override val enabledDeciderParam: Option[DeciderParam[Boolean]] = Some(CandidatePipeline.EnableInNetworkParam) override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) ) override val candidateSource: BaseCandidateSource[t.RecapQuery, t.CandidateTweet] = timelineRankerInNetworkCandidateSource override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, t.RecapQuery ] = TimelineRankerInNetworkQueryTransformer(identifier) override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[PipelineQuery, TweetCandidate, _] ] = Seq(RetweetSourceTweetFeatureHydrator) override def filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( RetweetSourceTweetRemovingFilter ) override val postFilterFeatureHydration: Seq[ BaseCandidateFeatureHydrator[PipelineQuery, TweetCandidate, _] ] = Seq(IsExtendedReplyFeatureHydrator, replyFeatureHydrator) override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[t.CandidateTweet] ] = Seq(ScoredTweetsInNetworkResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ t.CandidateTweet, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.tweet.get.id) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline import com.twitter.home_mixer.functional_component.feature_hydrator.ListIdsFeature import com.twitter.home_mixer.functional_component.feature_hydrator.TweetEntityServiceFeatureHydrator import com.twitter.home_mixer.functional_component.filter.ReplyFilter import com.twitter.home_mixer.functional_component.filter.RetweetFilter import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature import com.twitter.home_mixer.product.scored_tweets.candidate_source.ListTweet import com.twitter.home_mixer.product.scored_tweets.candidate_source.ListsCandidateSource import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ListNameFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsListsResponseFeatureTransformer import com.twitter.product_mixer.component_library.gate.EmptySeqFeatureGate import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelineservice.{thriftscala => t} import javax.inject.Inject import javax.inject.Singleton @Singleton class ScoredTweetsListsCandidatePipelineConfig @Inject() ( listsCandidateSource: ListsCandidateSource, listNameFeatureHydrator: ListNameFeatureHydrator, tweetEntityServiceFeatureHydrator: TweetEntityServiceFeatureHydrator) extends CandidatePipelineConfig[ ScoredTweetsQuery, Seq[t.TimelineQuery], ListTweet, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ScoredTweetsLists") private val MaxTweetsToFetchPerList = 20 override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( NonEmptySeqFeatureGate(ListIdsFeature), EmptySeqFeatureGate(CachedScoredTweetsFeature) ) override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, Seq[t.TimelineQuery] ] = { query => val listIds = query.features.map(_.get(ListIdsFeature)).get listIds.map { listId => t.TimelineQuery( timelineType = t.TimelineType.List, timelineId = listId, maxCount = MaxTweetsToFetchPerList.toShort, options = Some(t.TimelineQueryOptions(query.clientContext.userId)), timelineId2 = Some(t.TimelineId(t.TimelineType.List, listId, None)) ) } } override def candidateSource: CandidateSource[Seq[t.TimelineQuery], ListTweet] = listsCandidateSource override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[ListTweet] ] = Seq(ScoredTweetsListsResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ListTweet, TweetCandidate] = { sourceResult => TweetCandidate(id = sourceResult.tweet.statusId) } override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] ] = Seq( listNameFeatureHydrator, tweetEntityServiceFeatureHydrator, ) override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq(ReplyFilter, RetweetFilter) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsPopularVideosCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline import com.twitter.explore_ranker.{thriftscala => ert} import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsPopularVideosResponseFeatureTransformer import com.twitter.home_mixer.util.CachedScoredTweetsHelper import com.twitter.product_mixer.component_library.candidate_source.explore_ranker.ExploreRankerImmersiveRecsCandidateSource import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextMarshaller import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelines.configapi.decider.DeciderParam import javax.inject.Inject import javax.inject.Singleton @Singleton class ScoredTweetsPopularVideosCandidatePipelineConfig @Inject() ( exploreRankerCandidateSource: ExploreRankerImmersiveRecsCandidateSource, tweetypieStaticEntitiesFeatureHydrator: TweetypieStaticEntitiesFeatureHydrator) extends CandidatePipelineConfig[ ScoredTweetsQuery, ert.ExploreRankerRequest, ert.ExploreTweetRecommendation, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ScoredTweetsPopularVideos") private val MaxTweetsToFetch = 40 override val enabledDeciderParam: Option[DeciderParam[Boolean]] = Some(CandidatePipeline.EnablePopularVideosParam) override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) ) override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, ert.ExploreRankerRequest ] = { query => val excludedTweetIds = query.features.map( CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweets(_, identifier)) ert.ExploreRankerRequest( clientContext = ClientContextMarshaller(query.clientContext), product = ert.Product.HomeTimelineVideoInline, productContext = Some( ert.ProductContext.HomeTimelineVideoInline(ert.HomeTimelineVideoInline(excludedTweetIds))), maxResults = Some(MaxTweetsToFetch) ) } override def candidateSource: BaseCandidateSource[ ert.ExploreRankerRequest, ert.ExploreTweetRecommendation ] = exploreRankerCandidateSource override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[ert.ExploreTweetRecommendation] ] = Seq(ScoredTweetsPopularVideosResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ ert.ExploreTweetRecommendation, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] ] = Seq(tweetypieStaticEntitiesFeatureHydrator) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsStaticCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline import com.twitter.home_mixer.functional_component.feature_hydrator.TweetEntityServiceFeatureHydrator import com.twitter.home_mixer.functional_component.gate.AllowForYouRecommendationsGate import com.twitter.home_mixer.model.HomeFeatures.SignupCountryFeature import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsStaticCandidatePipelineConfig.Identifier import com.twitter.home_mixer.product.scored_tweets.candidate_source.StaticPostsCandidateSource import com.twitter.home_mixer.product.scored_tweets.candidate_source.StaticSourceRequest import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidateSourceParams import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsStaticResponseFeatureTransformer import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @Singleton class ScoredTweetsStaticCandidatePipelineConfig @Inject() ( staticPostsCandidateSource: StaticPostsCandidateSource, tweetEntityServiceFeatureHydrator: TweetEntityServiceFeatureHydrator) extends CandidatePipelineConfig[ ScoredTweetsQuery, StaticSourceRequest, Long, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = Identifier override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq(AllowForYouRecommendationsGate) override val supportedClientParam: Option[FSParam[Boolean]] = Some(CandidateSourceParams.EnableStaticSourceParam) override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, StaticSourceRequest ] = { query => val following = query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).getOrElse(Seq.empty).toSet val countryCode = query.clientContext.countryCode val signupCountryCode = query.features.flatMap(_.getOrElse(SignupCountryFeature, None)) val countryCodes = Seq(countryCode, signupCountryCode).flatten StaticSourceRequest(countryCodes = countryCodes, following = following) } override def candidateSource: CandidateSource[StaticSourceRequest, Long] = staticPostsCandidateSource override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] ] = Seq(tweetEntityServiceFeatureHydrator) override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[Long] ] = Seq(ScoredTweetsStaticResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ Long, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult) } } object ScoredTweetsStaticCandidatePipelineConfig { val SourceIdentifier = "ScoredTweetsStatic" val Identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier(SourceIdentifier) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline import com.twitter.home_mixer.functional_component.feature_hydrator.TweetEntityServiceFeatureHydrator import com.twitter.home_mixer.functional_component.gate.AllowForYouRecommendationsGate import com.twitter.home_mixer.product.scored_tweets.filter.ExtendedDirectedAtFilter import com.twitter.home_mixer.product.scored_tweets.filter.QualifiedRepliesFilter import com.twitter.tweet_mixer.{thriftscala => t} import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FetchParams import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsTweetMixerResponseFeatureTransformer import com.twitter.home_mixer.util.CachedScoredTweetsHelper import com.twitter.product_mixer.component_library.candidate_source.tweet_mixer.TweetMixerCandidateSource import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.module.TestUserMapper import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextMarshaller import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.request.ClientContext import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import javax.inject.Inject import javax.inject.Singleton /** * Candidate Pipeline Config that fetches tweets from TweetMixer. */ @Singleton class ScoredTweetsTweetMixerCandidatePipelineConfig @Inject() ( tweetMixerCandidateSource: TweetMixerCandidateSource, tweetEntityServiceFeatureHydrator: TweetEntityServiceFeatureHydrator, testUserMapper: TestUserMapper) extends CandidatePipelineConfig[ ScoredTweetsQuery, t.TweetMixerRequest, t.TweetResult, TweetCandidate ] { import ScoredTweetsTweetMixerCandidatePipelineConfig._ override val identifier: CandidatePipelineIdentifier = Identifier override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( AllowForYouRecommendationsGate, MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) ) override val candidateSource: BaseCandidateSource[t.TweetMixerRequest, t.TweetResult] = tweetMixerCandidateSource def getClientContext( query: ScoredTweetsQuery ): ClientContext = { if (testUserMapper.isTestUser(query.clientContext)) testUserMapper(query.clientContext) else query.clientContext } override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, t.TweetMixerRequest ] = { query => val excludedTweetIds = query.features.map( CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweets(_, identifier)) t.TweetMixerRequest( clientContext = ClientContextMarshaller(getClientContext(query)), product = t.Product.HomeRecommendedTweets, productContext = Some( t.ProductContext.HomeRecommendedTweetsProductContext( t.HomeRecommendedTweetsProductContext(excludedTweetIds = excludedTweetIds.map(_.toSet)))), maxResults = Some(query.params(FetchParams.TweetMixerMaxTweetsToFetchParam)) ) } override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] ] = Seq(tweetEntityServiceFeatureHydrator) override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( QualifiedRepliesFilter, ExtendedDirectedAtFilter ) override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[t.TweetResult] ] = Seq(ScoredTweetsTweetMixerResponseFeatureTransformer()) override val resultTransformer: CandidatePipelineResultsTransformer[ t.TweetResult, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } } object ScoredTweetsTweetMixerCandidatePipelineConfig { val Identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ScoredTweetsTweetMixer") } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsUtegCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerUtegQueryTransformer import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsUtegResponseFeatureTransformer import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerUtegCandidateSource import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.timelineranker.{thriftscala => t} import com.twitter.timelines.configapi.decider.DeciderParam import javax.inject.Inject import javax.inject.Singleton /** * Candidate Pipeline Config that fetches tweets from the Timeline Ranker UTEG Candidate Source */ @Singleton class ScoredTweetsUtegCandidatePipelineConfig @Inject() ( timelineRankerUtegCandidateSource: TimelineRankerUtegCandidateSource) extends CandidatePipelineConfig[ ScoredTweetsQuery, t.UtegLikedByTweetsQuery, t.CandidateTweet, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ScoredTweetsUteg") override val enabledDeciderParam: Option[DeciderParam[Boolean]] = Some(CandidatePipeline.EnableUtegParam) override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) ) override val candidateSource: BaseCandidateSource[t.UtegLikedByTweetsQuery, t.CandidateTweet] = timelineRankerUtegCandidateSource override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, t.UtegLikedByTweetsQuery ] = TimelineRankerUtegQueryTransformer(identifier) override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[t.CandidateTweet] ] = Seq(ScoredTweetsUtegResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ t.CandidateTweet, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.tweet.get.id) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", "src/thrift/com/twitter/search:earlybird-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsCommunitiesCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird import com.twitter.home_mixer.functional_component.feature_hydrator.TweetEntityServiceFeatureHydrator import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidateSourceParams import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.CommunitiesEarlybirdQueryTransformer import com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird.ScoredTweetsCommunitiesResponseFeatureTransformer import com.twitter.product_mixer.component_library.candidate_source.earlybird.EarlybirdTweetCandidateSource import com.twitter.product_mixer.component_library.feature_hydrator.candidate.communities.CommunityNamesFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.communities.CommunityMembershipsFeature import com.twitter.product_mixer.component_library.gate.EmptySeqFeatureGate import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.search.earlybird.{thriftscala => t} import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton @Singleton class ScoredTweetsCommunitiesCandidatePipelineConfig @Inject() ( earlybirdTweetCandidateSource: EarlybirdTweetCandidateSource, communityNamesFeatureHydrator: CommunityNamesFeatureHydrator, communitiesEarlybirdQueryTransformer: CommunitiesEarlybirdQueryTransformer, tweetEntityServiceFeatureHydrator: TweetEntityServiceFeatureHydrator) extends CandidatePipelineConfig[ ScoredTweetsQuery, t.EarlybirdRequest, t.ThriftSearchResult, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ScoredTweetsCommunities") override val supportedClientParam: Option[FSParam[Boolean]] = Some(CandidateSourceParams.EnableCommunitiesCandidateSourceParam) override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( NonEmptySeqFeatureGate(CommunityMembershipsFeature), EmptySeqFeatureGate(CachedScoredTweetsFeature) ) override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, t.EarlybirdRequest ] = communitiesEarlybirdQueryTransformer override def candidateSource: BaseCandidateSource[t.EarlybirdRequest, t.ThriftSearchResult] = earlybirdTweetCandidateSource override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[t.ThriftSearchResult] ] = Seq(ScoredTweetsCommunitiesResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ t.ThriftSearchResult, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.id) } override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] ] = Seq(tweetEntityServiceFeatureHydrator) override val postFilterFeatureHydration: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] ] = Seq(communityNamesFeatureHydrator) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdFrsCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird import com.twitter.finagle.thrift.ClientId import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FrsSeedUsersQueryFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdFrsQueryTransformer import com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird.ScoredTweetsEarlybirdFrsResponseFeatureTransformer import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.search.earlybird.{thriftscala => eb} import javax.inject.Inject import javax.inject.Singleton /** * Candidate Pipeline Config that fetches tweets from the earlybird FRS Candidate Source */ @Singleton class ScoredTweetsEarlybirdFrsCandidatePipelineConfig @Inject() ( earlybirdCandidateSource: EarlybirdCandidateSource, frsSeedUsersQueryFeatureHydrator: FrsSeedUsersQueryFeatureHydrator, clientId: ClientId) extends CandidatePipelineConfig[ ScoredTweetsQuery, eb.EarlybirdRequest, eb.ThriftSearchResult, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ScoredTweetsEarlybirdFrs") override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) ) override val queryFeatureHydration: Seq[ BaseQueryFeatureHydrator[ScoredTweetsQuery, _] ] = Seq(frsSeedUsersQueryFeatureHydrator) override val candidateSource: BaseCandidateSource[eb.EarlybirdRequest, eb.ThriftSearchResult] = earlybirdCandidateSource override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, eb.EarlybirdRequest ] = EarlybirdFrsQueryTransformer(identifier, clientId = Some(clientId.name)) override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[eb.ThriftSearchResult] ] = Seq(ScoredTweetsEarlybirdFrsResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ eb.ThriftSearchResult, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.id) } override def filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq.empty } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird import com.twitter.finagle.thrift.ClientId import com.twitter.home_mixer.functional_component.feature_hydrator.TweetEntityServiceFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.candidate_source.EarlybirdRealtimeCGTweetCandidateSource import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FollowedUserScoresFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.IsExtendedReplyFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.filter.ExtendedDirectedAtFilter import com.twitter.home_mixer.product.scored_tweets.filter.QualifiedRepliesFilter import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidateSourceParams import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdInNetworkQueryTransformer import com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird.ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.timelines.configapi.FSParam import javax.inject.Inject import javax.inject.Singleton /** * Candidate Pipeline Config that fetches tweets from the earlybird InNetwork Candidate Source */ @Singleton class ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig @Inject() ( earlybirdTweetCandidateSource: EarlybirdRealtimeCGTweetCandidateSource, followedUserScoresFeatureHydrator: FollowedUserScoresFeatureHydrator, tweetEntityServiceFeatureHydrator: TweetEntityServiceFeatureHydrator, clientId: ClientId) extends CandidatePipelineConfig[ ScoredTweetsQuery, eb.EarlybirdRequest, eb.ThriftSearchResult, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ScoredTweetsEarlybirdInNetwork") override val supportedClientParam: Option[FSParam[Boolean]] = Some( CandidateSourceParams.EnableInNetworkCandidateSourceParam) override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) ) override val queryFeatureHydration: Seq[ BaseQueryFeatureHydrator[ScoredTweetsQuery, _] ] = Seq( ParamGatedQueryFeatureHydrator( FeatureHydration.EnableFollowedUserScoreBackfillFeaturesParam, followedUserScoresFeatureHydrator ) ) override val candidateSource: BaseCandidateSource[eb.EarlybirdRequest, eb.ThriftSearchResult] = earlybirdTweetCandidateSource override val queryTransformer: CandidatePipelineQueryTransformer[ ScoredTweetsQuery, eb.EarlybirdRequest ] = EarlybirdInNetworkQueryTransformer(identifier, clientId = Some(clientId.name)) override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[eb.ThriftSearchResult] ] = Seq(ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer) override val preFilterFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] ] = Seq(tweetEntityServiceFeatureHydrator) override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( QualifiedRepliesFilter, ExtendedDirectedAtFilter ) override val postFilterFeatureHydration: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] ] = Seq(IsExtendedReplyFeatureHydrator) override val resultTransformer: CandidatePipelineResultsTransformer[ eb.ThriftSearchResult, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.id) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "servo/repo/src/main/scala", "src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/timelines/pinned_timelines:thrift-scala", "stitch/stitch-timelineservice", "strato/config/columns/content_understanding:content_understanding-strato-client", "strato/config/columns/timelines/pintweet:pintweet-strato-client", "strato/config/src/thrift/com/twitter/strato/columns/content_understanding:content_understanding-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/CachedScoredTweetsCandidateSource.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_source import com.twitter.home_mixer.util.CachedScoredTweetsHelper import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class CachedScoredTweetsCandidateSource @Inject() () extends CandidateSource[PipelineQuery, hmt.ScoredTweet] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("CachedScoredTweets") override def apply(request: PipelineQuery): Stitch[Seq[hmt.ScoredTweet]] = { Stitch.value( request.features.map(CachedScoredTweetsHelper.unseenCachedScoredTweets).getOrElse(Seq.empty)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ContentExplorationCandidateSource.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_source import com.twitter.home_mixer.product.scored_tweets.query_transformer.ContentExplorationQueryRequest import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.strato.catalog.Scan.Slice import com.twitter.strato.generated.client.content_understanding.CategoryColdStartPostsMhClientColumn import javax.inject.Inject import javax.inject.Singleton import scala.util.Random case class ContentExplorationCandidateResponse( tweetId: Long, tier: String) @Singleton class ContentExplorationCandidateSource @Inject() ( coldStartPostsColumn: CategoryColdStartPostsMhClientColumn) extends CandidateSource[ContentExplorationQueryRequest, ContentExplorationCandidateResponse] { private val MaxUserCategory = 5 private val MaxResultPerCategory = 100 private val MaxResult = 500 override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("ContentExploration") private val scanner = coldStartPostsColumn.scanner private def interweave( candidates: Seq[Seq[ContentExplorationCandidateResponse]] ): Seq[ContentExplorationCandidateResponse] = { if (candidates.isEmpty) return Seq.empty val maxLength = candidates.map(_.length).max (0 until maxLength) .flatMap { i => candidates.flatMap(seq => seq.lift(i)) }.take(MaxResult) } private def getSlice(): Slice[Long] = { val now = System.currentTimeMillis() val startId = SnowflakeId.firstIdFor(now - 6 * 60 * 60 * 1000L) val endId = SnowflakeId.firstIdFor(now) Slice[Long]( from = Some(startId), to = Some(endId), limit = Some(MaxResultPerCategory) ) } override def apply( request: ContentExplorationQueryRequest ): Stitch[Seq[ContentExplorationCandidateResponse]] = { val tags = request.userCategories val scan = getSlice() val version = request.version val scans: Seq[Stitch[Seq[ContentExplorationCandidateResponse]]] = tags.flatMap { tag => Seq( scanner.scan((version + "tier1_" + tag._1, scan)).map { results => Random .shuffle(results.map(_._1._2)) .map(id => ContentExplorationCandidateResponse(id, "tier1")) }, scanner.scan((version + "tier2_" + tag._1, scan)).map { results => Random .shuffle(results.map(_._1._2)) .map(id => ContentExplorationCandidateResponse(id, "tier2")) } ) } Stitch.collect(scans).map(interweave) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/EarlybirdRealtimeCGTweetCandidateSource.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_source import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRealtimCGEndpoint import com.twitter.product_mixer.component_library.candidate_source.earlybird.EarlybirdTweetCandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.search.earlybird.{thriftscala => t} import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class EarlybirdRealtimeCGTweetCandidateSource @Inject() ( @Named(EarlybirdRealtimCGEndpoint) earlybirdService: t.EarlybirdService.MethodPerEndpoint) extends EarlybirdTweetCandidateSource(earlybirdService) { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("EarlybirdRealtimeCGTweets") } @Singleton class EarlybirdNsfwCGTweetCandidateSource @Inject() ( @Named(EarlybirdRealtimCGEndpoint) earlybirdService: t.EarlybirdService.MethodPerEndpoint) extends EarlybirdTweetCandidateSource(earlybirdService) { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("EarlybirdNsfwCGTweets") } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_source import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.stitch.Stitch import com.twitter.stitch.timelineservice.TimelineService import com.twitter.timelineservice.{thriftscala => tls} import javax.inject.Inject import javax.inject.Singleton case class ListTweet(listId: Long, tweet: tls.Tweet) @Singleton class ListsCandidateSource @Inject() (timelineService: TimelineService) extends CandidateSource[Seq[tls.TimelineQuery], ListTweet] { override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("Lists") override def apply(requests: Seq[tls.TimelineQuery]): Stitch[Seq[ListTweet]] = Stitch .traverse(requests) { request => timelineService.getTimeline(request).map { response => response.entries.collect { case tls.TimelineEntry.Tweet(tweet) => ListTweet(response.timelineId.id, tweet) } } }.map(_.flatten) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/StaticPostsCandidateSource.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.candidate_source import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier import com.twitter.servo.cache.ExpiringLruInProcessCache import com.twitter.servo.cache.InProcessCache import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.strato.generated.client.timelines.pintweet.PinnedPostsClientColumn import com.twitter.timelines.pinned_timelines.{thriftscala => pt} import javax.inject.Inject import javax.inject.Singleton import scala.util.Random case class StaticSourceRequest(countryCodes: Seq[String], following: Set[Long]) object StaticPostsCandidateSource { private val BaseTTL = 30 private val StaticKey = 0 private val TTL = (BaseTTL + Random.nextInt(15)).minutes val cache: InProcessCache[Int, Option[pt.PinnedPost]] = new ExpiringLruInProcessCache(ttl = TTL, maximumSize = 100) } @Singleton class StaticPostsCandidateSource @Inject() (pinnedPostsClientColumn: PinnedPostsClientColumn) extends CandidateSource[StaticSourceRequest, Long] { import StaticPostsCandidateSource._ override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("StaticPosts") private val fetcher: Fetcher[Int, Unit, pt.PinnedPost] = pinnedPostsClientColumn.fetcher override def apply(request: StaticSourceRequest): Stitch[Seq[Long]] = { val postsStitch = cache.get(StaticKey).map(Stitch.value(_)).getOrElse { fetcher.fetch(StaticKey).map { response => cache.set(StaticKey, response.v) response.v } } postsStitch.map { _.map { post => val isInNetwork = request.following.contains(post.authorId) val countryMatches = request.countryCodes.exists { code => post.countryCodes.forall(_.map(_.toLowerCase).contains(code.toLowerCase)) } if (countryMatches && ((post.inNetwork && isInNetwork) || (post.outOfNetwork && !isInNetwork))) Seq(post.postId) else Seq.empty }.getOrElse(Seq.empty) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AncestorFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} import com.twitter.tweetconvosvc.{thriftscala => tcs} import com.twitter.util.Future import javax.inject.Inject import javax.inject.Singleton @Singleton class AncestorFeatureHydrator @Inject() ( conversationServiceClient: tcs.ConversationService.MethodPerEndpoint) extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Ancestor") override val features: Set[Feature[_, _]] = Set(AncestorsFeature) private val DefaultFeatureMap = FeatureMapBuilder().add(AncestorsFeature, Seq.empty).build() override def apply( query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap ): Stitch[FeatureMap] = OffloadFuturePools.offloadFuture { if (existingFeatures.getOrElse(InReplyToTweetIdFeature, None).isDefined) { val ancestorsRequest = tcs.GetAncestorsRequest(Seq(candidate.id)) conversationServiceClient.getAncestors(ancestorsRequest).map { getAncestorsResponse => val ancestors = getAncestorsResponse.ancestors.headOption .collect { case tcs.TweetAncestorsResult.TweetAncestors(ancestorsResult) if ancestorsResult.nonEmpty => ancestorsResult.head.ancestors ++ getTruncatedRootTweet(ancestorsResult.head) }.getOrElse(Seq.empty) FeatureMapBuilder().add(AncestorsFeature, ancestors).build() } } else Future.value(DefaultFeatureMap) } private def getTruncatedRootTweet( ancestors: ta.TweetAncestors, ): Option[ta.TweetAncestor] = { ancestors.conversationRootAuthorId.collect { case rootAuthorId if ancestors.state == ta.ReplyState.Partial && ancestors.ancestors.last.tweetId != ancestors.conversationId => ta.TweetAncestor(ancestors.conversationId, rootAuthorId) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.AuthorFeatureRepository import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.author_features.AuthorFeaturesAdapter import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.repository.KeyValueRepository import com.twitter.servo.repository.KeyValueResult import com.twitter.stitch.Stitch import com.twitter.timelines.author_features.v1.{thriftjava => af} import com.twitter.util.Future import com.twitter.util.Try import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object AuthorFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class AuthorFeatureHydrator @Inject() ( @Named(AuthorFeatureRepository) client: KeyValueRepository[Seq[Long], Long, af.AuthorFeatures], override val statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("AuthorFeature") override val features: Set[Feature[_, _]] = Set(AuthorFeature) override val statScope: String = identifier.toString override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val possiblyAuthorIds = extractKeys(candidates) val authorIds = possiblyAuthorIds.flatten val response: Future[KeyValueResult[Long, af.AuthorFeatures]] = if (authorIds.nonEmpty) client(authorIds) else Future.value(KeyValueResult.empty) response.map { result => possiblyAuthorIds.map { possiblyAuthorId => val value = observedGet(key = possiblyAuthorId, keyValueResult = result) val transformedValue = postTransformer(value) FeatureMapBuilder().add(AuthorFeature, transformedValue).build() } } } private def postTransformer(authorFeatures: Try[Option[af.AuthorFeatures]]): Try[DataRecord] = { authorFeatures.map { _.map { features => AuthorFeaturesAdapter.adaptToDataRecords(features).asScala.head } .getOrElse(new DataRecord()) } } private def extractKeys( candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[Long]] = { candidates.map { candidate => CandidatesUtil.getOriginalAuthorId(candidate.features) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorIsCreatorFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIsCreatorFeature import com.twitter.home_mixer.util.MissingKeyException import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.strato.generated.client.audiencerewards.audienceRewardsService.GetSuperFollowEligibilityOnUserClientColumn import com.twitter.util.Throw import javax.inject.Inject import javax.inject.Singleton @Singleton class AuthorIsCreatorFeatureHydrator @Inject() ( getSuperFollowEligibilityOnUserClientColumn: GetSuperFollowEligibilityOnUserClientColumn, statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("AuthorIsCreator") override val features: Set[Feature[_, _]] = Set(AuthorIsCreatorFeature) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") private val MissingKeyFeatureMap = FeatureMapBuilder() .add(AuthorIsCreatorFeature, Throw(MissingKeyException)) .build() private val DefaultFeatureMap = FeatureMapBuilder() .add(AuthorIsCreatorFeature, false) .build() override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { OffloadFuturePools.offloadStitch { val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)).distinct Stitch .collect { authorIds.map { authorId => getSuperFollowEligibilityOnUserClientColumn.fetcher .fetch(authorId) .map { authorId -> _.v } } }.map { authorIdsToIsCreator => val authorIdsToIsCreatorMap = authorIdsToIsCreator.toMap candidates.map { candidate => candidate.features.getOrElse(AuthorIdFeature, None) match { case Some(authorId) => authorIdsToIsCreatorMap.get(authorId) match { case Some(response) => keyFoundCounter.incr() FeatureMapBuilder() .add(AuthorIsCreatorFeature, response.getOrElse(false)).build() case _ => keyFailureCounter.incr() DefaultFeatureMap } case _ => MissingKeyFeatureMap } } } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "servo/repo/src/main/scala", "src/scala/com/twitter/timelines/prediction/adapters/conversation_features", "src/scala/com/twitter/timelines/prediction/common/adapters", "src/scala/com/twitter/timelines/prediction/features/semantic_core_features", "src/thrift/com/twitter/search:earlybird-scala", "strato/config/columns/content_understanding:content_understanding-strato-client", "strato/config/columns/lists/reads:core-strato-client", "strato/config/columns/recommendations/user-signal-service:user-signal-service-strato-client", "strato/config/columns/tweetypie/managed:managed-strato-client", "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", "timelines/src/main/scala/com/twitter/timelines/model/types", "tweetsource/common/src/main/thrift:thrift-scala", "user-signal-service/thrift/src/main/thrift:thrift-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.transport.Transport import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.param.HomeMixerInjectionNames.ScoredTweetsCache import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.TtlCache import com.twitter.stitch.Stitch import com.twitter.util.Return import com.twitter.util.Throw import com.twitter.util.Time import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton /** * Fetch scored Tweets from cache and exclude the expired ones */ @Singleton case class CachedScoredTweetsQueryFeatureHydrator @Inject() ( @Named(ScoredTweetsCache) scoredTweetsCache: TtlCache[Long, hmt.ScoredTweetsResponse]) extends QueryFeatureHydrator[PipelineQuery] with Conditionally[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("CachedScoredTweets") override val features: Set[Feature[_, _]] = Set(CachedScoredTweetsFeature) override def onlyIf(query: PipelineQuery): Boolean = { val serviceIdentifier = ServiceIdentifier.fromCertificate(Transport.peerCertificate) serviceIdentifier.role != "explore-mixer" && serviceIdentifier.role != "video-mixer" } override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId val tweetScoreTtl = query.params(CachedScoredTweets.TTLParam) Stitch.callFuture(scoredTweetsCache.get(Seq(userId))).map { keyValueResult => keyValueResult(userId) match { case Return(cachedCandidatesOpt) => val cachedScoredTweets = cachedCandidatesOpt.map(_.scoredTweets).getOrElse(Seq.empty) val nonExpiredTweets = cachedScoredTweets.filter { tweet => tweet.lastScoredTimestampMs.exists(Time.fromMilliseconds(_).untilNow < tweetScoreTtl) } FeatureMap(CachedScoredTweetsFeature, nonExpiredTweets) case Throw(exception) => throw exception } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.UserLanguagesFeature import com.twitter.home_mixer.model.HomeFeatures.DeviceLanguageFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdSearchResultFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRepository import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird.EarlybirdAdapter import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.home_mixer.util.earlybird.EarlybirdResponseUtil import com.twitter.ml.api.DataRecord import com.twitter.ml.api.RichDataRecord import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.search.common.constants.{thriftscala => scc} import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import com.twitter.timelines.earlybird.common.utils.InNetworkEngagement import com.twitter.util.Future import com.twitter.util.Return import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object EarlybirdDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class EarlybirdFeatureHydrator @Inject() ( @Named(EarlybirdRepository) client: KeyValueRepository[ (Seq[Long], Long), Long, eb.ThriftSearchResult ], override val statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Earlybird") override val features: Set[Feature[_, _]] = Set( EarlybirdDataRecordFeature, EarlybirdFeature, EarlybirdSearchResultFeature, SourceTweetEarlybirdFeature, TweetUrlsFeature ) override val statScope: String = identifier.toString private val scopedStatsReceiver = statsReceiver.scope(statScope) private val originalKeyFoundCounter = scopedStatsReceiver.counter("originalKey/found") private val originalKeyNotFoundCounter = scopedStatsReceiver.counter("originalKey/notFound") private val ebFeaturesNotExistPredicate: CandidateWithFeatures[TweetCandidate] => Boolean = candidate => candidate.features.getOrElse(EarlybirdFeature, None).isEmpty private val InternalUrlPattern = "" override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val candidatesWithoutExistingEBFeatures = candidates .filter { candidate => val isEmpty = ebFeaturesNotExistPredicate(candidate) if (isEmpty) originalKeyNotFoundCounter.incr() else originalKeyFoundCounter.incr() isEmpty } val (candidatesWithSearchFeatures, candidatesWithoutSearchFeatures) = candidatesWithoutExistingEBFeatures.partition { candidate => candidate.features.getOrElse(EarlybirdSearchResultFeature, None).nonEmpty } // hydrate both candidates and their sources val candidateIdsToHydrate = ( candidatesWithSearchFeatures .filter(_.features.getOrElse(IsRetweetFeature, false)) .map(CandidatesUtil.getOriginalTweetId) ++ candidatesWithoutSearchFeatures.map(_.candidate.id) ++ candidatesWithoutSearchFeatures.map(CandidatesUtil.getOriginalTweetId) ).distinct client((candidateIdsToHydrate, query.getRequiredUserId)) .flatMap( handleResponse(query, candidates, _, candidateIdsToHydrate, candidatesWithSearchFeatures)) } private[feature_hydrator] def getFeatureMap( candidate: CandidateWithFeatures[TweetCandidate], query: PipelineQuery, idToSearchResults: Map[Long, eb.ThriftSearchResult], screenName: Option[String], userLanguages: Seq[scc.ThriftLanguage], uiLanguage: Option[scc.ThriftLanguage], tweetCountByAuthorId: Map[Long, Int], followedUserIds: Set[Long], mutuallyFollowedUserIds: Set[Long], inNetworkEngagement: InNetworkEngagement, ): FeatureMap = { val candidateIsRetweet = candidate.features.getOrElse(IsRetweetFeature, false) val existingEbFeatures = candidate.features.getOrElse(EarlybirdFeature, None) val existingSourceTweetEbFeatures = candidate.features.getOrElse(SourceTweetEarlybirdFeature, None) val tweetEbFeatures = if (existingEbFeatures.nonEmpty) existingEbFeatures else { idToSearchResults.get(candidate.candidate.id).map { searchResult => EarlybirdResponseUtil.getThriftTweetFeaturesFromSearchResult( searcherUserId = query.getRequiredUserId, screenName, userLanguages, uiLanguage, tweetCountByAuthorId, followedUserIds, mutuallyFollowedUserIds, idToSearchResults, inNetworkEngagement, searchResult ) } } val sourceTweetEbFeatures = if (candidateIsRetweet) { if (existingSourceTweetEbFeatures.nonEmpty) existingSourceTweetEbFeatures else { idToSearchResults.get(CandidatesUtil.getOriginalTweetId(candidate)).map { searchResult => EarlybirdResponseUtil.getThriftTweetFeaturesFromSearchResult( searcherUserId = query.getRequiredUserId, screenName, userLanguages, uiLanguage, tweetCountByAuthorId, followedUserIds, mutuallyFollowedUserIds, idToSearchResults, inNetworkEngagement, searchResult ) } } } else None val originalTweetEbFeatures = if (sourceTweetEbFeatures.nonEmpty) sourceTweetEbFeatures else tweetEbFeatures val earlybirdDataRecord = EarlybirdAdapter.adaptToDataRecords(originalTweetEbFeatures).asScala.head val tesUrls = candidate.features.getOrElse(TweetUrlsFeature, Seq.empty) val urls = if (tesUrls.isEmpty) tweetEbFeatures.flatMap(_.urlsList).getOrElse(Seq.empty) else tesUrls val rdr = new RichDataRecord(earlybirdDataRecord) FeatureMapBuilder(sizeHint = 5) .add(EarlybirdFeature, tweetEbFeatures) .add(EarlybirdDataRecordFeature, rdr.getRecord) .add(EarlybirdSearchResultFeature, idToSearchResults.get(candidate.candidate.id)) .add(SourceTweetEarlybirdFeature, sourceTweetEbFeatures) .add(TweetUrlsFeature, urls) .build() } private def handleResponse( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]], results: KeyValueResult[Long, eb.ThriftSearchResult], candidateIdsToHydrate: Seq[Long], candidatesWithSearchFeatures: Seq[CandidateWithFeatures[TweetCandidate]], ): Future[Seq[FeatureMap]] = { val batchSize = 64 val queryFeatureMap = query.features.getOrElse(FeatureMap.empty) val userLanguages = queryFeatureMap.getOrElse(UserLanguagesFeature, Seq.empty) val uiLanguageCode = queryFeatureMap.getOrElse(DeviceLanguageFeature, None) val screenName = queryFeatureMap.getOrElse(UserScreenNameFeature, None) val followedUserIds = queryFeatureMap.getOrElse(SGSFollowedUsersFeature, Seq.empty).toSet val mutuallyFollowedUserIds = queryFeatureMap.getOrElse(SGSMutuallyFollowedUsersFeature, Seq.empty).toSet val searchResults = candidateIdsToHydrate .map { id => observedGet(Some(id), results) }.collect { case Return(Some(value)) => value } val allSearchResults = searchResults ++ candidatesWithSearchFeatures.flatMap { candidate => candidate.features.getOrElse(EarlybirdSearchResultFeature, None) } val idToSearchResults = allSearchResults.map { searchResults => searchResults.id -> searchResults }.toMap val uiLanguage = EarlybirdResponseUtil.getLanguage(uiLanguageCode) val tweetCountByAuthorId = EarlybirdResponseUtil.getTweetCountByAuthorId(allSearchResults) val inNetworkEngagement = InNetworkEngagement(followedUserIds.toSeq, mutuallyFollowedUserIds, allSearchResults) OffloadFuturePools .offloadBatchElementToElement[CandidateWithFeatures[TweetCandidate], FeatureMap]( candidates, candidate => getFeatureMap( candidate = candidate, query = query, idToSearchResults = idToSearchResults, screenName = screenName, userLanguages = userLanguages, uiLanguage = uiLanguage, tweetCountByAuthorId = tweetCountByAuthorId, followedUserIds = followedUserIds, mutuallyFollowedUserIds = mutuallyFollowedUserIds, inNetworkEngagement = inNetworkEngagement ), batchSize ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FollowedUserScoresFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton object FollowedUserScoresFeature extends Feature[PipelineQuery, Map[Long, Double]] @Singleton case class FollowedUserScoresFeatureHydrator @Inject() () extends QueryFeatureHydrator[PipelineQuery] { private val UpdateFollowedUserThreshold = 1000 private val MaxFollowedUsers = 1500 private val DefaultAuthorScore = 2D override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FollowedUserScores") override def features: Set[Feature[_, _]] = Set(FollowedUserScoresFeature) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val sgsFollowedUserIds = query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).toSeq.flatten val sgsFollowedUserIdSet = sgsFollowedUserIds.toSet val authorScoreMap = query.features .map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double])) .getOrElse(Map.empty) val filteredAuthorScoreMap = authorScoreMap.filter { entry => sgsFollowedUserIdSet.contains(entry._1) } val updatedSgsFollowedUserIds = if (sgsFollowedUserIds.size >= UpdateFollowedUserThreshold) { filteredAuthorScoreMap.keySet ++ sgsFollowedUserIds .filter(!filteredAuthorScoreMap.keySet.contains(_)) .take(Math.max(0, MaxFollowedUsers - filteredAuthorScoreMap.size)) } else sgsFollowedUserIds val existingAuthorIdScores = filteredAuthorScoreMap.values.toList.sorted val imputedScoreIndex = Math.min(existingAuthorIdScores.length - 1, (existingAuthorIdScores.length * 0.5f).toInt) val imputedScore = if (imputedScoreIndex >= 0) existingAuthorIdScores(imputedScoreIndex) else DefaultAuthorScore val updatedAuthorScoreMap = updatedSgsFollowedUserIds .map(_ -> imputedScore).toMap ++ filteredAuthorScoreMap Stitch.value { FeatureMapBuilder() .add(FollowedUserScoresFeature, updatedAuthorScoreMap) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.follow_recommendations.{thriftscala => frs} import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.product_mixer.component_library.candidate_source.recommendations.UserFollowRecommendationsCandidateSource import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton object FrsSeedUserIdsFeature extends Feature[TweetCandidate, Option[Seq[Long]]] object FrsUserToFollowedByUserIdsFeature extends Feature[TweetCandidate, Map[Long, Seq[Long]]] @Singleton case class FrsSeedUsersQueryFeatureHydrator @Inject() ( userFollowRecommendationsCandidateSource: UserFollowRecommendationsCandidateSource) extends QueryFeatureHydrator[ScoredTweetsQuery] { private val maxUsersToFetch = 100 override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FrsSeedUsers") override def features: Set[Feature[_, _]] = Set( FrsSeedUserIdsFeature, FrsUserToFollowedByUserIdsFeature ) override def hydrate(query: ScoredTweetsQuery): Stitch[FeatureMap] = { val frsRequest = frs.RecommendationRequest( clientContext = frs.ClientContext(query.getOptionalUserId), displayLocation = frs.DisplayLocation.HomeTimelineTweetRecs, maxResults = Some(maxUsersToFetch) ) userFollowRecommendationsCandidateSource(StratoKeyView(frsRequest, Unit)) .map { userRecommendations: Seq[frs.UserRecommendation] => val seedUserIds = userRecommendations.map(_.userId) val seedUserIdsSet = seedUserIds.toSet val userToFollowedByUserIds: Map[Long, Seq[Long]] = userRecommendations.flatMap { userRecommendation => if (seedUserIdsSet.contains(userRecommendation.userId)) { val followProof = userRecommendation.reason.flatMap(_.accountProof).flatMap(_.followProof) val followedByUserIds = followProof.map(_.userIds).getOrElse(Seq.empty) Some(userRecommendation.userId -> followedByUserIds) } else { None } }.toMap FeatureMapBuilder() .add(FrsSeedUserIdsFeature, Some(seedUserIds)) .add(FrsUserToFollowedByUserIdsFeature, userToFollowedByUserIds) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GizmoduckAuthorFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.conversions.DurationOps._ import com.twitter.ads.entities.db.{thriftscala => ae} import com.twitter.gizmoduck.{thriftscala => gt} import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIsProtectedFeature import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.IsSupportAccountReplyFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.snowflake.id.SnowflakeId import javax.inject.Inject import javax.inject.Singleton @Singleton class GizmoduckAuthorFeatureHydrator @Inject() (gizmoduck: gt.UserService.MethodPerEndpoint) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GizmoduckAuthor") override val features: Set[Feature[_, _]] = Set(AuthorIsBlueVerifiedFeature, AuthorIsProtectedFeature, IsSupportAccountReplyFeature) private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.AdvertiserAccount, gt.QueryFields.Profile, gt.QueryFields.Safety) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)) val response = gizmoduck.get( userIds = authorIds.distinct, queryFields = queryFields, context = gt.LookupContext() ) response.map { hydratedAuthors => val userMetadataMap = hydratedAuthors .collect { case userResult if userResult.user.isDefined => val user = userResult.user.get val blueVerified = user.safety.flatMap(_.isBlueVerified).getOrElse(false) val isProtected = user.safety.exists(_.isProtected) (user.id, (blueVerified, isProtected)) }.toMap.withDefaultValue((false, false)) candidates.map { candidate => val authorId = candidate.features.get(AuthorIdFeature).get val (isBlueVerified, isProtected) = userMetadataMap(authorId) // Some accounts run promotions on Twitter and send replies automatically. // We assume that a reply that took more than one minute is not an auto-reply. // If time difference doesn't exist, this means that one of the tweets was // not snowflake and therefore much older, and therefore OK as an extended reply. val timeDifference = candidate.features.getOrElse(InReplyToTweetIdFeature, None).map { SnowflakeId.timeFromId(candidate.candidate.id) - SnowflakeId.timeFromId(_) } val isAutoReply = timeDifference.exists(_ < 1.minute) FeatureMapBuilder() .add(AuthorIsBlueVerifiedFeature, isBlueVerified) .add(AuthorIsProtectedFeature, isProtected) .add(IsSupportAccountReplyFeature, isAutoReply) .build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GraphTwoHopFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.graph_feature_service.{thriftscala => gfs} import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.GraphTwoHopRepository import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.two_hop_features.TwoHopFeaturesAdapter import com.twitter.util.Try import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object GraphTwoHopFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class GraphTwoHopFeatureHydrator @Inject() ( @Named(GraphTwoHopRepository) client: KeyValueRepository[(Seq[Long], Long), Long, Seq[ gfs.IntersectionValue ]], override val statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GraphTwoHop") override val features: Set[Feature[_, _]] = Set(GraphTwoHopFeature, FollowedByUserIdsFeature) override val statScope: String = identifier.toString private val twoHopFeaturesAdapter = new TwoHopFeaturesAdapter private val FollowFeatureType = gfs.FeatureType(gfs.EdgeType.Following, gfs.EdgeType.FollowedBy) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { // Apply filters to in network candidates for retweets only. val (inNetworkCandidates, oonCandidates) = candidates.partition { candidate => candidate.features.getOrElse(FromInNetworkSourceFeature, false) } val inNetworkCandidatesToHydrate = inNetworkCandidates.filter(_.features.getOrElse(IsRetweetFeature, false)) val candidatesToHydrate = (inNetworkCandidatesToHydrate ++ oonCandidates) .flatMap(candidate => CandidatesUtil.getOriginalAuthorId(candidate.features)).distinct val response = client((candidatesToHydrate, query.getRequiredUserId)) response.map { result => candidates.map { candidate => val originalAuthorId = CandidatesUtil.getOriginalAuthorId(candidate.features) val value = observedGet(key = originalAuthorId, keyValueResult = result) val transformedValue = postTransformer(value) val followedByUserIds = value.toOption .flatMap(getFollowedByUserIds(_)) .getOrElse(Seq.empty) FeatureMapBuilder() .add(GraphTwoHopFeature, transformedValue) .add(FollowedByUserIdsFeature, followedByUserIds) .build() } } } private def getFollowedByUserIds(input: Option[Seq[gfs.IntersectionValue]]): Option[Seq[Long]] = input.map(_.filter(_.featureType == FollowFeatureType).flatMap(_.intersectionIds).flatten) private def postTransformer(input: Try[Option[Seq[gfs.IntersectionValue]]]): Try[DataRecord] = input.map(twoHopFeaturesAdapter.adaptToDataRecords(_).asScala.head) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/InvalidateCachedScoredTweetsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature import com.twitter.home_mixer.model.HomeFeatures.LastNonPollingTimeFeature import com.twitter.home_mixer.model.HomeFeatures.UserLastExplicitSignalTimeFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Invalidates (flushes) cached ScoredTweets upon detecting explicit signal from user made since * previous request */ object InvalidateCachedScoredTweetsQueryFeatureHydrator extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( "InvalidateCachedScoredTweets") override val features: Set[Feature[_, _]] = Set(CachedScoredTweetsFeature) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val lastRequestTime = query.features.getOrElse(FeatureMap.empty).getOrElse(LastNonPollingTimeFeature, None) val lastExplicitSignalTime = query.features .getOrElse(FeatureMap.empty).getOrElse(UserLastExplicitSignalTimeFeature, None) val hasRecentExplicitSignal = lastExplicitSignalTime .flatMap { engagementTime => lastRequestTime.map { requestTime => engagementTime > requestTime } }.getOrElse(false) val featureMap = if (hasRecentExplicitSignal) { FeatureMap(CachedScoredTweetsFeature, Seq.empty) } else { val cachedTweets = query.features.getOrElse(FeatureMap.empty).getOrElse(CachedScoredTweetsFeature, Seq.empty) FeatureMap(CachedScoredTweetsFeature, cachedTweets) } Stitch.value(featureMap) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsColdStartPostFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.IsColdStartPostFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.IsColdStartPostInMemCache import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.InProcessCache import com.twitter.strato.generated.client.content_understanding.ColdStartPostsMetadataMhClientColumn import com.twitter.stitch.Stitch import com.twitter.strato.columns.content_understanding.content_exploration.thriftscala.ColdStartPostStatus import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class IsColdStartPostFeatureHydrator @Inject() ( coldStartPostsMetadataMhClientColumn: ColdStartPostsMetadataMhClientColumn, @Named(IsColdStartPostInMemCache) isColdStartPostInMemCache: InProcessCache[ Long, Boolean ], statsReceiver: StatsReceiver) extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("IsColdStartPost") override val features: Set[Feature[_, _]] = Set(IsColdStartPostFeature) private val DefaultFeatureMap = FeatureMap(IsColdStartPostFeature, false) private val ineligibleStatusSet: Set[ColdStartPostStatus] = Set(ColdStartPostStatus.Tier1Ineligible, ColdStartPostStatus.Tier1IneligibleHighQuality) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val cacheHitCounter = scopedStatsReceiver.counter("cache/hit") private val cacheMissCounter = scopedStatsReceiver.counter("cache/miss") private val storeHitCounter = scopedStatsReceiver.counter("store/hit") private val storeMissCounter = scopedStatsReceiver.counter("store/miss") private val fetchExceptionCounter = scopedStatsReceiver.counter("fetch/exception") override def apply( query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap ): Stitch[FeatureMap] = { val postId = candidate.id isColdStartPostInMemCache .get(postId) .map { cachedValue => cacheHitCounter.incr() Stitch.value(FeatureMap(IsColdStartPostFeature, cachedValue)) }.getOrElse { cacheMissCounter.incr() coldStartPostsMetadataMhClientColumn.fetcher .fetch(postId) .map { response => if (response.v.isDefined) { storeHitCounter.incr() } else { storeMissCounter.incr() } val isColdStartPost = response.v.flatMap(_.status) match { case Some(status) => !ineligibleStatusSet.contains(status) case _ => false } isColdStartPostInMemCache.set(postId, isColdStartPost) FeatureMap(IsColdStartPostFeature, isColdStartPost) }.handle { case _: Exception => fetchExceptionCounter.incr() DefaultFeatureMap } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsExtendedReplyFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsExtendedReplyFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch object IsExtendedReplyFeatureHydrator extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("IsExtendedReply") override def features: Set[Feature[_, _]] = Set(IsExtendedReplyFeature) private val TrueFeatureMap = FeatureMapBuilder().add(IsExtendedReplyFeature, true).build() private val FalseFeatureMap = FeatureMapBuilder().add(IsExtendedReplyFeature, false).build() override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { val followedUsers = query.features.map(_.get(SGSFollowedUsersFeature)).getOrElse(Seq.empty).toSet candidates.map { candidate => val features = candidate.features val isExtendedReply = features.getOrElse(InReplyToTweetIdFeature, None).nonEmpty && !features.getOrElse(IsRetweetFeature, false) && features.getOrElse(InReplyToUserIdFeature, None).exists(!followedUsers.contains(_)) if (isExtendedReply) TrueFeatureMap else FalseFeatureMap } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListIdsQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.socialgraph.{thriftscala => sg} import com.twitter.stitch.Stitch import com.twitter.stitch.socialgraph.SocialGraph import javax.inject.Inject import javax.inject.Singleton case object ListIdsFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { override val defaultValue: Seq[Long] = Seq.empty } @Singleton class ListIdsQueryFeatureHydrator @Inject() (socialGraph: SocialGraph) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ListIds") override val features: Set[Feature[_, _]] = Set(ListIdsFeature) private val MaxListsToFetch = 20 override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId val ownedSubscribedRequest = sg.IdsRequest( relationships = Seq( sg.SrcRelationship(userId, sg.RelationshipType.ListIsSubscriber, hasRelationship = true), sg.SrcRelationship(userId, sg.RelationshipType.ListOwning, hasRelationship = true) ), pageRequest = Some(sg.PageRequest(selectAll = Some(false), count = Some(MaxListsToFetch))), context = Some( sg.LookupContext( includeInactive = false, performUnion = Some(true), includeAll = Some(false) ) ) ) socialGraph.ids(ownedSubscribedRequest).map { response => FeatureMapBuilder().add(ListIdsFeature, response.ids).build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListNameFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.ListIdFeature import com.twitter.home_mixer.model.HomeFeatures.ListNameFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.socialgraph.{thriftscala => sg} import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.strato.generated.client.lists.reads.CoreOnListClientColumn import javax.inject.Inject import javax.inject.Singleton @Singleton class ListNameFeatureHydrator @Inject() (coreOnListClientColumn: CoreOnListClientColumn) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ListName") override val features: Set[Feature[_, _]] = Set(ListNameFeature) private val fetcher: Fetcher[Long, Unit, sg.SocialgraphList] = coreOnListClientColumn.fetcher override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { val listIds = candidates.flatMap(_.features.getOrElse(ListIdFeature, None)).distinct val listIdNameMapStitch = Stitch.collect { listIds.map { listId => listId -> fetcher.fetch(listId).map(_.v.map(_.name)) }.toMap } listIdNameMapStitch.map { listIdNameMap => candidates.map { candidate => val listId = candidate.features.getOrElse(ListIdFeature, None) val listName = listId.flatMap(listIdNameMap.get).flatten FeatureMapBuilder().add(ListNameFeature, listName).build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/LowSignalUserQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.LowSignalUserMaxSignalCount import com.twitter.home_mixer.util.SignalUtil import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.client.Fetcher import com.twitter.strato.generated.client.recommendations.user_signal_service.SignalsClientColumn import com.twitter.usersignalservice.{thriftscala => uss} import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton @Singleton case class LowSignalUserQueryFeatureHydrator @Inject() ( signalsClientColumn: SignalsClientColumn) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("LowSignalUser") override val features: Set[Feature[_, _]] = Set( LowSignalUserFeature, UserRecentEngagementTweetIdsFeature, UserLastExplicitSignalTimeFeature) val fetcher: Fetcher[SignalsClientColumn.Key, Unit, SignalsClientColumn.Value] = signalsClientColumn.fetcher val MaxFetch = 15L val MinSignalFavCount = 0 val MaxSignalFavCount = 5000000 val LowSignalUserMaxSignalAge = 90.days private def getTimestamp(signal: uss.Signal): Option[Time] = { if (signal.timestamp == 0L) None else Some(Time.fromMilliseconds(signal.timestamp)) } override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val signalRequests = SignalUtil.ExplicitSignals.map { signal => uss.SignalRequest( maxResults = Some(MaxFetch), signalType = signal, minFavCount = Some(MinSignalFavCount), maxFavCount = Some(MaxSignalFavCount) ) } val batchSignalRequest = uss.BatchSignalRequest( userId = query.getRequiredUserId, signalRequest = signalRequests, clientId = Some(uss.ClientIdentifier.CrMixerHome) ) fetcher.fetch(batchSignalRequest).map { response => val signals = response.v.map(_.signalResponse.values.toSeq.flatten).getOrElse(Seq.empty) val tweetIds = signals.collect { case signal if signal.targetInternalId.isDefined => signal.targetInternalId.get match { case id: com.twitter.simclusters_v2.thriftscala.InternalId.TweetId => Some(id.tweetId.toLong) case _ => None } }.flatten val timeFilteredSignals = signals.filter { signal => getTimestamp(signal).exists { signalTime => Time.now.since(signalTime) < LowSignalUserMaxSignalAge } } val mostRecentTimestamp = timeFilteredSignals.flatMap(getTimestamp).reduceOption((a, b) => if (a > b) a else b) val lowSignalUser = timeFilteredSignals.size < query.params(LowSignalUserMaxSignalCount) FeatureMapBuilder() .add(LowSignalUserFeature, lowSignalUser) .add(UserRecentEngagementTweetIdsFeature, tweetIds.toList) .add(UserLastExplicitSignalTimeFeature, mostRecentTimestamp) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/MetricCenterUserCountingFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.MetricCenterUserCountingFeatureRepository import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.onboarding.relevance.features.{thriftjava => rf} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import com.twitter.util.Future import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton object MetricCenterUserCountingFeature extends Feature[TweetCandidate, Option[rf.MCUserCountingFeatures]] @Singleton class MetricCenterUserCountingFeatureHydrator @Inject() ( @Named(MetricCenterUserCountingFeatureRepository) client: KeyValueRepository[Seq[ Long ], Long, rf.MCUserCountingFeatures], override val statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("MetricCenterUserCounting") override val features: Set[Feature[_, _]] = Set(MetricCenterUserCountingFeature) override val statScope: String = identifier.toString override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val possiblyAuthorIds = extractKeys(candidates) val userIds = possiblyAuthorIds.flatten val response: Future[KeyValueResult[Long, rf.MCUserCountingFeatures]] = if (userIds.isEmpty) Future.value(KeyValueResult.empty) else client(userIds) response.map { result => possiblyAuthorIds.map { possiblyAuthorId => val value = observedGet(key = possiblyAuthorId, keyValueResult = result) FeatureMapBuilder().add(MetricCenterUserCountingFeature, value).build() } } } private def extractKeys( candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[Long]] = { candidates.map { candidate => candidate.features .getTry(AuthorIdFeature) .toOption .flatten } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphFeatureRepository import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.repository.Repository import com.twitter.timelines.real_graph.{thriftscala => rg} import com.twitter.stitch.Stitch import com.twitter.timelines.model.UserId import com.twitter.timelines.real_graph.v1.thriftscala.RealGraphEdgeFeatures import com.twitter.user_session_store.{thriftscala => uss} import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton object RealGraphFeatures extends Feature[PipelineQuery, Option[Map[UserId, RealGraphEdgeFeatures]]] @Singleton class RealGraphQueryFeatureHydrator @Inject() ( @Named(RealGraphFeatureRepository) repository: Repository[Long, Option[uss.UserSession]]) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RealGraphFeatures") override val features: Set[Feature[_, _]] = Set(RealGraphFeatures) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { Stitch.callFuture { repository(query.getRequiredUserId).map { userSession => val realGraphFeaturesMap = userSession.flatMap { userSession => userSession.realGraphFeatures.collect { case rg.RealGraphFeatures.V1(realGraphFeatures) => val edgeFeatures = realGraphFeatures.edgeFeatures ++ realGraphFeatures.oonEdgeFeatures edgeFeatures.map { edge => edge.destId -> edge }.toMap } } FeatureMapBuilder().add(RealGraphFeatures, realGraphFeaturesMap).build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures import com.twitter.home_mixer.util.MissingKeyException import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter import com.twitter.timelines.prediction.adapters.real_graph.RealGraphFeaturesAdapter import com.twitter.timelines.real_graph.v1.{thriftscala => v1} import com.twitter.timelines.real_graph.{thriftscala => rg} import com.twitter.util.Throw import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ object RealGraphViewerAuthorDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object RealGraphViewerAuthorsDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class RealGraphViewerAuthorFeatureHydrator @Inject() () extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RealGraphViewerAuthor") override val features: Set[Feature[_, _]] = Set(RealGraphViewerAuthorDataRecordFeature, RealGraphViewerAuthorsDataRecordFeature) private val realGraphEdgeFeaturesAdapter = new RealGraphFeaturesAdapter private val realGraphEdgeFeaturesCombineAdapter = new RealGraphEdgeFeaturesCombineAdapter(prefix = "authors.realgraph") private val MissingKeyFeatureMap = FeatureMapBuilder() .add(RealGraphViewerAuthorDataRecordFeature, Throw(MissingKeyException)) .add(RealGraphViewerAuthorsDataRecordFeature, Throw(MissingKeyException)) .build() override def apply( query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap ): Stitch[FeatureMap] = OffloadFuturePools.offload { val viewerId = query.getRequiredUserId val realGraphFeatures = query.features .flatMap(_.getOrElse(RealGraphFeatures, None)) .getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures]) existingFeatures.getOrElse(AuthorIdFeature, None) match { case Some(authorId) => val realGraphAuthorFeatures = getRealGraphViewerAuthorFeatures(viewerId, authorId, realGraphFeatures) val realGraphAuthorDataRecord = realGraphEdgeFeaturesAdapter .adaptToDataRecords(realGraphAuthorFeatures).asScala.headOption.getOrElse(new DataRecord) val combinedRealGraphFeaturesDataRecord = for { inReplyToAuthorId <- existingFeatures.getOrElse(InReplyToUserIdFeature, None) } yield { val combinedRealGraphFeatures = getCombinedRealGraphFeatures(Seq(authorId, inReplyToAuthorId), realGraphFeatures) realGraphEdgeFeaturesCombineAdapter .adaptToDataRecords(Some(combinedRealGraphFeatures)).asScala.headOption .getOrElse(new DataRecord) } FeatureMapBuilder() .add(RealGraphViewerAuthorDataRecordFeature, realGraphAuthorDataRecord) .add( RealGraphViewerAuthorsDataRecordFeature, combinedRealGraphFeaturesDataRecord.getOrElse(new DataRecord)) .build() case _ => MissingKeyFeatureMap } } private def getRealGraphViewerAuthorFeatures( viewerId: Long, authorId: Long, realGraphEdgeFeaturesMap: Map[Long, v1.RealGraphEdgeFeatures] ): rg.UserRealGraphFeatures = { realGraphEdgeFeaturesMap.get(authorId) match { case Some(realGraphEdgeFeatures) => rg.UserRealGraphFeatures( srcId = viewerId, features = rg.RealGraphFeatures.V1( v1.RealGraphFeatures(edgeFeatures = Seq(realGraphEdgeFeatures)))) case _ => rg.UserRealGraphFeatures( srcId = viewerId, features = rg.RealGraphFeatures.V1(v1.RealGraphFeatures(edgeFeatures = Seq.empty))) } } } object RealGraphViewerAuthorFeatureHydrator { def getCombinedRealGraphFeatures( userIds: Seq[Long], realGraphEdgeFeaturesMap: Map[Long, v1.RealGraphEdgeFeatures] ): rg.RealGraphFeatures = { val edgeFeatures = userIds.flatMap(realGraphEdgeFeaturesMap.get) rg.RealGraphFeatures.V1(v1.RealGraphFeatures(edgeFeatures = edgeFeatures)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter import com.twitter.timelines.real_graph.v1.{thriftscala => v1} import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ object RealGraphViewerRelatedUsersDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class RealGraphViewerRelatedUsersFeatureHydrator @Inject() () extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RealGraphViewerRelatedUsers") override val features: Set[Feature[_, _]] = Set(RealGraphViewerRelatedUsersDataRecordFeature) private val RealGraphEdgeFeaturesCombineAdapter = new RealGraphEdgeFeaturesCombineAdapter override def apply( query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap ): Stitch[FeatureMap] = OffloadFuturePools.offload { val realGraphQueryFeatures = query.features .flatMap(_.getOrElse(RealGraphFeatures, None)) .getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures]) val allRelatedUserIds = getRelatedUserIds(existingFeatures) val realGraphFeatures = RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures( allRelatedUserIds, realGraphQueryFeatures ) val realGraphFeaturesDataRecord = RealGraphEdgeFeaturesCombineAdapter .adaptToDataRecords(Some(realGraphFeatures)).asScala.headOption .getOrElse(new DataRecord) FeatureMapBuilder() .add(RealGraphViewerRelatedUsersDataRecordFeature, realGraphFeaturesDataRecord) .build() } private def getRelatedUserIds(features: FeatureMap): Seq[Long] = { (CandidatesUtil.getEngagerUserIds(features) ++ features.getOrElse(AuthorIdFeature, None) ++ features.getOrElse(MentionUserIdFeature, Seq.empty) ++ features.getOrElse(SourceUserIdFeature, None) ++ features.getOrElse(DirectedAtUserIdFeature, None)).distinct } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.adapters.realtime_interaction_graph.RealTimeInteractionGraphFeaturesAdapter import com.twitter.timelines.prediction.features.realtime_interaction_graph.RealTimeInteractionGraphEdgeFeatures import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ object RealTimeInteractionGraphEdgeFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class RealTimeInteractionGraphEdgeFeatureHydrator @Inject() () extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RealTimeInteractionGraphEdge") override val features: Set[Feature[_, _]] = Set(RealTimeInteractionGraphEdgeFeature) private val realTimeInteractionGraphFeaturesAdapter = new RealTimeInteractionGraphFeaturesAdapter override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { val userVertex = query.features.flatMap(_.getOrElse(RealTimeInteractionGraphUserVertexQueryFeature, None)) val realTimeInteractionGraphFeaturesMap = userVertex.map(RealTimeInteractionGraphEdgeFeatures(_, Time.now)) candidates.map { candidate => val feature = candidate.features.getOrElse(AuthorIdFeature, None).flatMap { authorId => realTimeInteractionGraphFeaturesMap.flatMap(_.get(authorId)) } val dataRecordFeature = realTimeInteractionGraphFeaturesAdapter.adaptToDataRecords(feature).asScala.head FeatureMapBuilder().add(RealTimeInteractionGraphEdgeFeature, dataRecordFeature).build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionGraphUserVertexCache import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.ReadCache import com.twitter.stitch.Stitch import com.twitter.wtf.real_time_interaction_graph.{thriftscala => ig} import javax.inject.Inject import javax.inject.Singleton object RealTimeInteractionGraphUserVertexQueryFeature extends Feature[PipelineQuery, Option[ig.UserVertex]] @Singleton class RealTimeInteractionGraphUserVertexQueryFeatureHydrator @Inject() ( @Named(RealTimeInteractionGraphUserVertexCache) client: ReadCache[Long, ig.UserVertex], override val statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RealTimeInteractionGraphUserVertex") override val features: Set[Feature[_, _]] = Set(RealTimeInteractionGraphUserVertexQueryFeature) override val statScope: String = identifier.toString override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId Stitch.callFuture( client.get(Seq(userId)).map { results => val feature = observedGet(key = Some(userId), keyValueResult = results) FeatureMapBuilder() .add(RealTimeInteractionGraphUserVertexQueryFeature, feature) .build() } ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieContentDataRecordFeature import com.twitter.home_mixer.functional_component.feature_hydrator.WithDefaultFeatureMap import com.twitter.home_mixer.model.ContentFeatures import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content.InReplyToContentFeatureAdapter import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird.InReplyToEarlybirdAdapter import com.twitter.home_mixer.util.ReplyRetweetUtil import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.timelines.conversation_features.v1.thriftscala.ConversationFeatures import com.twitter.timelines.conversation_features.{thriftscala => cf} import com.twitter.timelines.prediction.adapters.conversation_features.ConversationFeaturesAdapter import com.twitter.util.Duration import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ object InReplyToTweetHydratedEarlybirdFeature extends Feature[TweetCandidate, Option[ThriftTweetFeatures]] object ConversationDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object InReplyToEarlybirdDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object InReplyToTweetypieContentDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } /** * The purpose of this hydrator is to * 1) hydrate simple features into replies and their ancestor tweets * 2) keep both the normal replies and ancestor source candidates, but hydrate into the candidates * features useful for predicting the quality of the replies and source ancestor tweets. */ @Singleton class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with WithDefaultFeatureMap { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ReplyTweet") override val features: Set[Feature[_, _]] = Set( ConversationDataRecordFeature, InReplyToTweetHydratedEarlybirdFeature, InReplyToEarlybirdDataRecordFeature, InReplyToTweetypieContentDataRecordFeature ) private val defaultDataRecord: DataRecord = new DataRecord() override val defaultFeatureMap = FeatureMap( ConversationDataRecordFeature, defaultDataRecord, InReplyToTweetHydratedEarlybirdFeature, None, InReplyToEarlybirdDataRecordFeature, defaultDataRecord, InReplyToTweetypieContentDataRecordFeature, defaultDataRecord ) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val hydratedReplyCounter = scopedStatsReceiver.counter("hydratedReply") private val hydratedAncestorCounter = scopedStatsReceiver.counter("hydratedAncestor") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { // only hydrate for IN candidates val eligibleCandidates = candidates.filter(_.features.getOrElse(FromInNetworkSourceFeature, false)) val replyToInReplyToTweetMap = ReplyRetweetUtil.replyTweetIdToInReplyToTweetMap(eligibleCandidates) val candidatesWithRepliesHydrated = candidates.map { candidate => replyToInReplyToTweetMap .get(candidate.candidate.id).map { inReplyToTweet => hydratedReplyCounter.incr() hydratedReplyCandidate(candidate, inReplyToTweet) }.getOrElse((candidate, None, None, defaultDataRecord)) } /** * Update ancestor tweets with descendant replies and hydrate simple features from one of * the descendants. */ val ancestorTweetToDescendantRepliesMap = ReplyRetweetUtil.ancestorTweetIdToDescendantRepliesMap(eligibleCandidates) val candidatesWithRepliesAndAncestorTweetsHydrated = candidatesWithRepliesHydrated.map { case ( maybeAncestorTweetCandidate, updatedReplyConversationFeatures, inReplyToTweetEarlyBirdFeature, inReplyToTweetContentDataRecord ) => ancestorTweetToDescendantRepliesMap .get(maybeAncestorTweetCandidate.candidate.id) .map { descendantReplies => hydratedAncestorCounter.incr() val (ancestorTweetCandidate, updatedConversationFeatures): ( CandidateWithFeatures[TweetCandidate], Option[ConversationFeatures] ) = hydrateAncestorTweetCandidate( maybeAncestorTweetCandidate, descendantReplies, updatedReplyConversationFeatures) ( ancestorTweetCandidate, inReplyToTweetEarlyBirdFeature, updatedConversationFeatures, inReplyToTweetContentDataRecord) } .getOrElse( ( maybeAncestorTweetCandidate, inReplyToTweetEarlyBirdFeature, updatedReplyConversationFeatures, inReplyToTweetContentDataRecord )) } candidatesWithRepliesAndAncestorTweetsHydrated.map { case ( candidate, inReplyToTweetEarlyBirdFeature, updatedConversationFeatures, inReplyToTweetContentDataRecord) => val conversationDataRecordFeature = updatedConversationFeatures .map(f => ConversationFeaturesAdapter.adaptToDataRecord(cf.ConversationFeatures.V1(f))) .getOrElse(defaultDataRecord) val inReplyToEarlybirdDataRecord = InReplyToEarlybirdAdapter .adaptToDataRecords(inReplyToTweetEarlyBirdFeature).asScala.head val inReplyToContentDataRecord = { if (inReplyToTweetContentDataRecord.equals(defaultDataRecord)) { InReplyToContentFeatureAdapter .adaptToDataRecords( inReplyToTweetEarlyBirdFeature.map(ContentFeatures.fromThrift)).asScala.head } else inReplyToTweetContentDataRecord } FeatureMap( ConversationDataRecordFeature, conversationDataRecordFeature, InReplyToTweetHydratedEarlybirdFeature, inReplyToTweetEarlyBirdFeature, InReplyToEarlybirdDataRecordFeature, inReplyToEarlybirdDataRecord, InReplyToTweetypieContentDataRecordFeature, inReplyToContentDataRecord ) case _ => defaultFeatureMap } } private def hydratedReplyCandidate( replyCandidate: CandidateWithFeatures[TweetCandidate], inReplyToTweetCandidate: CandidateWithFeatures[TweetCandidate] ): ( CandidateWithFeatures[TweetCandidate], Option[ConversationFeatures], Option[ThriftTweetFeatures], DataRecord ) = { val tweetedAfterInReplyToTweetInSecs = ( originalTweetAgeFromSnowflake(inReplyToTweetCandidate), originalTweetAgeFromSnowflake(replyCandidate)) match { case (Some(inReplyToTweetAge), Some(replyTweetAge)) => Some((inReplyToTweetAge - replyTweetAge).inSeconds.toLong) case _ => None } val existingConversationFeatures = Some( replyCandidate.features .getOrElse(ConversationFeature, None).getOrElse(ConversationFeatures())) val updatedConversationFeatures = existingConversationFeatures match { case Some(v1) => Some( v1.copy( tweetedAfterInReplyToTweetInSecs = tweetedAfterInReplyToTweetInSecs, isSelfReply = Some( replyCandidate.features.getOrElse( AuthorIdFeature, None) == inReplyToTweetCandidate.features.getOrElse(AuthorIdFeature, None)) ) ) case _ => None } // Note: if inReplyToTweet is a retweet, we need to read early bird feature from the merged // early bird feature field from RetweetSourceTweetFeatureHydrator class. // But if inReplyToTweet is a reply, we return its early bird feature directly val sourceFeatures = inReplyToTweetCandidate.features .getOrElse(SourceTweetEarlybirdFeature, None) val inReplyToTweetThriftTweetFeaturesOpt = { if (inReplyToTweetCandidate.features.getOrElse(IsRetweetFeature, false) && sourceFeatures.nonEmpty) { sourceFeatures } else { inReplyToTweetCandidate.features.getOrElse(EarlybirdFeature, None) } } val inReplyToTweetContentDataRecord = inReplyToTweetCandidate.features .getOrElse(TweetypieContentDataRecordFeature, defaultDataRecord) ( replyCandidate, updatedConversationFeatures, inReplyToTweetThriftTweetFeaturesOpt, inReplyToTweetContentDataRecord) } private def hydrateAncestorTweetCandidate( ancestorTweetCandidate: CandidateWithFeatures[TweetCandidate], descendantReplies: Seq[CandidateWithFeatures[TweetCandidate]], updatedReplyConversationFeatures: Option[ConversationFeatures] ): (CandidateWithFeatures[TweetCandidate], Option[ConversationFeatures]) = { // Ancestor could be a reply. For example, in thread: tweetA -> tweetB -> tweetC, // tweetB is a reply and ancestor at the same time. Hence, tweetB's conversation feature // will be updated by hydratedReplyCandidate and hydrateAncestorTweetCandidate functions. val existingConversationFeatures = if (updatedReplyConversationFeatures.nonEmpty) updatedReplyConversationFeatures else Some( ancestorTweetCandidate.features .getOrElse(ConversationFeature, None).getOrElse(ConversationFeatures())) val updatedConversationFeatures = existingConversationFeatures match { case Some(v1) => Some( v1.copy( hasDescendantReplyCandidate = Some(true), hasInNetworkDescendantReply = Some(descendantReplies.exists(_.features.getOrElse(InNetworkFeature, false))) )) case _ => None } (ancestorTweetCandidate, updatedConversationFeatures) } private def originalTweetAgeFromSnowflake( candidate: CandidateWithFeatures[TweetCandidate] ): Option[Duration] = { SnowflakeId .timeFromIdOpt( candidate.features .getOrElse(SourceTweetIdFeature, None).getOrElse(candidate.candidate.id)) .map(Time.now - _) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RequestTimeQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.FollowingLastNonPollingTimeFeature import com.twitter.home_mixer.model.HomeFeatures.LastNonPollingTimeFeature import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature import com.twitter.ml.api.DataRecord import com.twitter.ml.api.util.FDsl._ import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.ACCOUNT_AGE_INTERVAL import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.IS_12_MONTH_NEW_USER import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.IS_30_DAY_NEW_USER import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_BETWEEN_NON_POLLING_REQUESTS_AVG import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_SINCE_LAST_NON_POLLING_REQUEST import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.USER_ID_IS_SNOWFLAKE_ID import com.twitter.user_session_store.ReadRequest import com.twitter.user_session_store.ReadWriteUserSessionStore import com.twitter.user_session_store.UserSessionDataset import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset import com.twitter.util.Time import javax.inject.Inject import javax.inject.Singleton object RequestTimeDataRecordFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton case class RequestTimeQueryFeatureHydrator @Inject() ( userSessionStore: ReadWriteUserSessionStore) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RequestTime") override val features: Set[Feature[_, _]] = Set( FollowingLastNonPollingTimeFeature, LastNonPollingTimeFeature, NonPollingTimesFeature, RequestTimeDataRecordFeature ) private val datasets: Set[UserSessionDataset] = Set(UserSessionDataset.NonPollingTimes) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { userSessionStore .read(ReadRequest(query.getRequiredUserId, datasets)) .map { userSession => val nonPollingTimestamps = userSession.flatMap(_.nonPollingTimestamps) val lastNonPollingTime = nonPollingTimestamps .flatMap(_.nonPollingTimestampsMs.headOption) .map(Time.fromMilliseconds) val followingLastNonPollingTime = nonPollingTimestamps .flatMap(_.mostRecentHomeLatestNonPollingTimestampMs) .map(Time.fromMilliseconds) val nonPollingTimes = nonPollingTimestamps .map(_.nonPollingTimestampsMs) .getOrElse(Seq.empty) val requestTimeDataRecord = getRequestTimeDataRecord(query, nonPollingTimes) FeatureMapBuilder() .add(FollowingLastNonPollingTimeFeature, followingLastNonPollingTime) .add(LastNonPollingTimeFeature, lastNonPollingTime) .add(NonPollingTimesFeature, nonPollingTimes) .add(RequestTimeDataRecordFeature, requestTimeDataRecord) .build() } } def getRequestTimeDataRecord(query: PipelineQuery, nonPollingTimes: Seq[Long]): DataRecord = { val requestTimeMs = query.queryTime.inMillis val accountAge = SnowflakeId.timeFromIdOpt(query.getRequiredUserId) val timeSinceAccountCreation = accountAge.map(query.queryTime.since) val timeSinceEarliestNonPollingRequest = nonPollingTimes.lastOption.map(requestTimeMs - _) val timeSinceLastNonPollingRequest = nonPollingTimes.headOption.map(requestTimeMs - _) new DataRecord() .setFeatureValue(USER_ID_IS_SNOWFLAKE_ID, accountAge.isDefined) .setFeatureValue( IS_30_DAY_NEW_USER, timeSinceAccountCreation.map(_ < 30.days).getOrElse(false) ) .setFeatureValue( IS_12_MONTH_NEW_USER, timeSinceAccountCreation.map(_ < 365.days).getOrElse(false) ) .setFeatureValueFromOption( ACCOUNT_AGE_INTERVAL, timeSinceAccountCreation.flatMap(AccountAgeInterval.fromDuration).map(_.id.toLong) ) .setFeatureValueFromOption( TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS, timeSinceAccountCreation.map(_.inSeconds.toDouble) ) .setFeatureValueFromOption( TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, timeSinceEarliestNonPollingRequest.map(_.toDouble / math.max(1.0, nonPollingTimes.size)) ) .setFeatureValueFromOption( TIME_SINCE_LAST_NON_POLLING_REQUEST, timeSinceLastNonPollingRequest.map(_.toDouble) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RetweetSourceTweetFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerInNetworkSourceTweetsByTweetIdMapFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures import com.twitter.stitch.Stitch import com.twitter.timelineranker.thriftscala.CandidateTweet object SourceTweetEarlybirdFeature extends Feature[TweetCandidate, Option[ThriftTweetFeatures]] /** * Feature Hydrator that bulk hydrates source tweets' features to retweet candidates */ object RetweetSourceTweetFeatureHydrator extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RetweetSourceTweet") override val features: Set[Feature[_, _]] = Set( SourceTweetEarlybirdFeature, ) private val DefaultFeatureMap = FeatureMapBuilder() .add(SourceTweetEarlybirdFeature, None) .build() override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { val sourceTweetsByTweetId: Option[Map[Long, CandidateTweet]] = { query.features.map( _.getOrElse( TimelineRankerInNetworkSourceTweetsByTweetIdMapFeature, Map.empty[Long, CandidateTweet])) } /** * Return DefaultFeatureMap (no-op to candidate) when it is unfeasible to hydrate the * source tweet's feature to the current candidate: early bird does not return source * tweets info / candidate is not a retweet / sourceTweetId is not found */ Stitch.value { if (sourceTweetsByTweetId.exists(_.nonEmpty)) { candidates.map { candidate => val candidateIsRetweet = candidate.features.getOrElse(IsRetweetFeature, false) val sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None) if (!candidateIsRetweet || sourceTweetId.isEmpty) { DefaultFeatureMap } else { val sourceTweet = sourceTweetsByTweetId.flatMap(_.get(sourceTweetId.get)) if (sourceTweet.nonEmpty) { val source = sourceTweet.get FeatureMapBuilder() .add(SourceTweetEarlybirdFeature, source.features) .build() } else { DefaultFeatureMap } } } } else { candidates.map(_ => DefaultFeatureMap) } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SGSMutuallyFollowedUserHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.socialgraph.{thriftscala => sg} import com.twitter.stitch.Stitch import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient} import javax.inject.Inject import javax.inject.Singleton object SGSMutuallyFollowedUsersFeature extends Feature[PipelineQuery, Seq[Long]] @Singleton case class SGSMutuallyFollowedUserHydrator @Inject() ( socialGraphStitchClient: SocialGraphStitchClient) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("SGSMutuallyFollowedUsers") override val features: Set[Feature[_, _]] = Set(SGSMutuallyFollowedUsersFeature) private val SocialGraphLimit = 14999 private val MaxFollowTargets = 1500 private val DefaultFeatureMap = FeatureMapBuilder() .add(SGSMutuallyFollowedUsersFeature, Seq.empty) .build() override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val sgsFollowedUserIds = query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).toSeq.flatten if (sgsFollowedUserIds.nonEmpty) { val mutuallyFollowedRequest = sg.IdsRequest( relationships = Seq( sg.SrcRelationship( query.getRequiredUserId, sg.RelationshipType.FollowedBy, hasRelationship = true, targets = Some(sgsFollowedUserIds.take(MaxFollowTargets)) ), ), pageRequest = Some(sg.PageRequest(count = Some(SocialGraphLimit))) ) socialGraphStitchClient.ids(mutuallyFollowedRequest).map(_.ids).map { mutuallyFollowedUsers => FeatureMapBuilder() .add(SGSMutuallyFollowedUsersFeature, mutuallyFollowedUsers) .build() } } else Stitch.value(DefaultFeatureMap) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SemanticCoreFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.functional_component.feature_hydrator.RealTimeEntityRealGraphFeatures import com.twitter.home_mixer.functional_component.feature_hydrator.WithDefaultFeatureMap import com.twitter.home_mixer.model.HomeFeatures.SemanticAnnotationIdsFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableRealTimeEntityRealGraphFeaturesParam import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.util.SRichDataRecord import com.twitter.ml.api.{Feature => mlFeature} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.recos.entities.{thriftscala => ent} import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.features.semantic_core_features.SemanticCoreFeatures import com.twitter.wtf.entity_real_graph.{thriftscala => erg} object SemanticCoreDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object SemanticCoreFeatureHydrator extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] with WithDefaultFeatureMap { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("SemanticCore") override val features: Set[Feature[_, _]] = Set(SemanticCoreDataRecordFeature) override val defaultFeatureMap: FeatureMap = FeatureMap(SemanticCoreDataRecordFeature, SemanticCoreDataRecordFeature.defaultValue) override def onlyIf(query: PipelineQuery): Boolean = query.params(EnableRealTimeEntityRealGraphFeaturesParam) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]], ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { val viewerErgFeatures = query.features.flatMap(_.getOrElse(RealTimeEntityRealGraphFeatures, None)) candidates.map { candidate => val candidateEntities = candidate.features.getOrElse(SemanticAnnotationIdsFeature, Seq.empty) val engagements: Seq[Map[erg.EngagementType, erg.Features]] = { viewerErgFeatures .map { engagementsByEntity => engagementsByEntity.collect { case (ent.Entity.SemanticCore(sc), featuresByEngagementType) if candidateEntities.contains(sc.entityId) => featuresByEngagementType }.toSeq }.getOrElse(Seq.empty) } val rawFeaturesByEngagementType: Map[erg.EngagementType, Seq[erg.Features]] = SemanticCoreFeatures.engagementTypes.map { engagementType => val features = engagements.flatMap(_.get(engagementType)) engagementType -> features }.toMap val decayedScoreFeatureValues: Map[mlFeature[_], List[Double]] = SemanticCoreFeatures.decayedScoreFeatures.mapValues { engagementType => val valuesOpt: Option[Seq[Double]] = rawFeaturesByEngagementType.get(engagementType).map(_.map(_.scores.oneDayHalfLife)) valuesOpt match { case Some(values) => values.toList case None => List.empty[Double] } } val normalizedCountFeatureValues: Map[mlFeature[_], List[Double]] = SemanticCoreFeatures.normalizedCountFeatures.mapValues { engagementType => val valuesOpt: Option[Seq[Double]] = rawFeaturesByEngagementType.get(engagementType).map(_.flatMap(_.normalizedCount)) valuesOpt match { case Some(values) => values.toList case None => List.empty[Double] } } buildFeatureMap(decayedScoreFeatureValues ++ normalizedCountFeatureValues) } } private def buildFeatureMap( featureValuesMap: Map[mlFeature[_], List[Double]] ): FeatureMap = { val featureContext = new FeatureContext(SemanticCoreFeatures.outputFeaturesPostMerge.toSeq: _*) val richRecord = new SRichDataRecord(new DataRecord, featureContext) SemanticCoreFeatures.hydrateCountFeatures( richRecord = richRecord, features = SemanticCoreFeatures.precomputedCountFeatures, featureValuesMap = featureValuesMap ) FeatureMap(SemanticCoreDataRecordFeature, richRecord.getRecord) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.clients.strato.twistly.SimClustersRecentEngagementSimilarityClient import com.twitter.timelines.configapi.decider.BooleanDeciderParam import com.twitter.timelines.prediction.adapters.twistly.SimClustersRecentEngagementSimilarityFeaturesAdapter import javax.inject.Inject import javax.inject.Singleton object SimClustersEngagementSimilarityFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class SimClustersEngagementSimilarityFeatureHydrator @Inject() ( simClustersEngagementSimilarityClient: SimClustersRecentEngagementSimilarityClient) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("SimClustersEngagementSimilarity") override val features: Set[Feature[_, _]] = Set(SimClustersEngagementSimilarityFeature) private val simClustersRecentEngagementSimilarityFeaturesAdapter = new SimClustersRecentEngagementSimilarityFeaturesAdapter override def onlyIf(query: PipelineQuery): Boolean = { val param: BooleanDeciderParam = ScoredTweetsParam.EnableSimClustersSimilarityFeatureHydrationDeciderParam query.params.apply(param) } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val tweetToCandidates = candidates.map(candidate => candidate.candidate.id -> candidate).toMap val tweetIds = tweetToCandidates.keySet.toSeq val userId = query.getRequiredUserId val userTweetEdges = tweetIds.map(tweetId => (userId, tweetId)) simClustersEngagementSimilarityClient .getSimClustersRecentEngagementSimilarityScores(userTweetEdges).map { simClustersRecentEngagementSimilarityScoresMap => candidates.map { candidate => val similarityFeatureOpt = simClustersRecentEngagementSimilarityScoresMap .get(userId -> candidate.candidate.id).flatten val dataRecordOpt = similarityFeatureOpt.map { similarityFeature => simClustersRecentEngagementSimilarityFeaturesAdapter .adaptToDataRecords(similarityFeature) .get(0) } FeatureMapBuilder() .add(SimClustersEngagementSimilarityFeature, dataRecordOpt.getOrElse(new DataRecord)) .build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersUserTweetScoresHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.dal.personal_data.{thriftjava => pd} import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.strato.catalog.Fetch import com.twitter.strato.generated.client.ml.featureStore.SimClustersUserInterestedInTweetEmbeddingDotProduct20M145K2020OnUserTweetEdgeClientColumn import javax.inject.Inject import javax.inject.Singleton object SimClustersUserInterestedInTweetEmbeddingDataRecordFeature extends DataRecordOptionalFeature[TweetCandidate, Double] with DoubleDataRecordCompatible { override val featureName: String = "user-tweet.recommendations.sim_clusters_scores.user_interested_in_tweet_embedding_dot_product_20m_145k_2020" override val personalDataTypes: Set[pd.PersonalDataType] = Set(pd.PersonalDataType.InferredInterests) } @Singleton class SimClustersUserTweetScoresHydrator @Inject() ( simClustersColumn: SimClustersUserInterestedInTweetEmbeddingDotProduct20M145K2020OnUserTweetEdgeClientColumn, statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("SimClustersUserTweetScores") override val features: Set[Feature[_, _]] = Set( SimClustersUserInterestedInTweetEmbeddingDataRecordFeature) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyLossCounter = scopedStatsReceiver.counter("key/loss") private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") private val keySkipCounter = scopedStatsReceiver.counter("key/skip") private val DefaultFeatureMap = FeatureMapBuilder() .add(SimClustersUserInterestedInTweetEmbeddingDataRecordFeature, None) .build() private val MinFavToHydrate = 9 override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { Stitch.run { Stitch.collect { candidates.map { candidate => val ebFeatures = candidate.features.getOrElse(EarlybirdFeature, None) val favCount = ebFeatures.flatMap(_.favCountV2).getOrElse(0) if (ebFeatures.isEmpty || favCount >= MinFavToHydrate) { simClustersColumn.fetcher .fetch((query.getRequiredUserId, candidate.candidate.id), Unit) .map { case Fetch.Result(response, _) => if (response.nonEmpty) keyFoundCounter.incr() else keyLossCounter.incr() FeatureMapBuilder() .add(SimClustersUserInterestedInTweetEmbeddingDataRecordFeature, response) .build() case _ => keyFailureCounter.incr() DefaultFeatureMap } } else { keySkipCounter.incr() Stitch.value(DefaultFeatureMap) } } } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TSPInferredTopicFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.contentrecommender.{thriftscala => cr} import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.clients.strato.topics.TopicSocialProofClient import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => sid} import com.twitter.topiclisting.TopicListingViewerContext import com.twitter.tsp.{thriftscala => tsp} import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]] object TSPInferredTopicDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TSPInferredTopicFeatureHydrator @Inject() ( topicSocialProofClient: TopicSocialProofClient) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic") override val features: Set[Feature[_, _]] = Set( TSPInferredTopicFeature, TSPInferredTopicDataRecordFeature, TopicIdSocialContextFeature, TopicContextFunctionalityTypeFeature ) private val topK = 3 private val SourcesToSetSocialProof: Set[sid.CandidateTweetSourceId] = Set(sid.CandidateTweetSourceId.Simcluster) private val DefaultFeatureMap = FeatureMapBuilder() .add(TSPInferredTopicFeature, Map.empty[Long, Double]) .add(TSPInferredTopicDataRecordFeature, new DataRecord()) .add(TopicIdSocialContextFeature, None) .add(TopicContextFunctionalityTypeFeature, None) .build() override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val tags = candidates.collect { case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn => candidate.candidate.id -> candidate.features .getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag]) }.toMap val topicSocialProofRequest = tsp.TopicSocialProofRequest( userId = query.getRequiredUserId, tweetIds = candidates.map(_.candidate.id).toSet, displayLocation = cr.DisplayLocation.HomeTimeline, topicListingSetting = tsp.TopicListingSetting.Followable, context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift, bypassModes = None, // Only TweetMixer source has this data. Convert the TweetMixer metric tag to tsp metric tag. tags = if (tags.isEmpty) None else Some(tags) ) topicSocialProofClient .getTopicTweetSocialProofResponse(topicSocialProofRequest) .map { case Some(response) => handleResponse(response, candidates) case _ => candidates.map { _ => DefaultFeatureMap } } } private def handleResponse( response: tsp.TopicSocialProofResponse, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[FeatureMap] = { candidates.map { candidate => val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty) if (topicWithScores.nonEmpty) { val (socialProofId, socialProofFunctionalityType) = if (candidate.features .getOrElse(CandidateSourceIdFeature, None) .exists(SourcesToSetSocialProof.contains)) { getSocialProof(topicWithScores) } else (None, None) val inferredTopicFeatures = topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap val inferredTopicDataRecord = InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head FeatureMapBuilder() .add(TSPInferredTopicFeature, inferredTopicFeatures) .add(TSPInferredTopicDataRecordFeature, inferredTopicDataRecord) .add(TopicIdSocialContextFeature, socialProofId) .add(TopicContextFunctionalityTypeFeature, socialProofFunctionalityType) .build() } else DefaultFeatureMap } } private def getSocialProof( topicWithScores: Seq[tsp.TopicWithScore] ): (Option[Long], Option[TopicContextFunctionalityType]) = { val followingTopicId = topicWithScores.collectFirst { case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) => topicId } if (followingTopicId.nonEmpty) { return (followingTopicId, Some(BasicTopicContextFunctionalityType)) } val implicitFollowingId = topicWithScores.collectFirst { case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) => topicId } if (implicitFollowingId.nonEmpty) { return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType)) } (None, None) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetMetaDataFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.ml.api.DataRecord import com.twitter.ml.api.RichDataRecord import com.twitter.ml.api.constant.SharedFeatures import com.twitter.ml.api.util.DataRecordConverters._ import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import java.lang.{Long => JLong} object TweetMetaDataDataRecord extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object TweetMetaDataFeatureHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetMetaData") override def features: Set[Feature[_, _]] = Set(TweetMetaDataDataRecord) override def apply( query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap ): Stitch[FeatureMap] = OffloadFuturePools.offload { val richDataRecord = new RichDataRecord() setFeatures(richDataRecord, candidate, existingFeatures) FeatureMapBuilder().add(TweetMetaDataDataRecord, richDataRecord.getRecord).build() } private def setFeatures( richDataRecord: RichDataRecord, candidate: TweetCandidate, existingFeatures: FeatureMap ): Unit = { richDataRecord.setFeatureValue[JLong](SharedFeatures.TWEET_ID, candidate.id) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.ORIGINAL_AUTHOR_ID, CandidatesUtil.getOriginalAuthorId(existingFeatures)) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.CANDIDATE_TWEET_SOURCE_ID, existingFeatures.getOrElse(CandidateSourceIdFeature, None).map(_.value.toLong)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetTimeFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.ml.api.DataRecord import com.twitter.ml.api.util.FDsl._ import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._ import com.twitter.util.Duration import scala.collection.Searching._ object TweetTimeDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object TweetTimeFeatureHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetTime") override val features: Set[Feature[_, _]] = Set(TweetTimeDataRecordFeature) override def apply( query: PipelineQuery, candidate: TweetCandidate, existingFeatures: FeatureMap ): Stitch[FeatureMap] = { val tweetFeatures = existingFeatures.getOrElse(EarlybirdFeature, None) val timeSinceTweetCreation = SnowflakeId.timeFromIdOpt(candidate.id).map(query.queryTime.since) val timeSinceTweetCreationMs = timeSinceTweetCreation.map(_.inMillis) val timeSinceSourceTweetCreationOpt = existingFeatures .getOrElse(SourceTweetIdFeature, None) .flatMap { sourceTweetId => SnowflakeId.timeFromIdOpt(sourceTweetId).map(query.queryTime.since) }.orElse(timeSinceTweetCreation) val lastFavSinceCreationHrs = tweetFeatures.flatMap(_.lastFavSinceCreationHrs).map(_.toDouble) val lastRetweetSinceCreationHrs = tweetFeatures.flatMap(_.lastRetweetSinceCreationHrs).map(_.toDouble) val lastReplySinceCreationHrs = tweetFeatures.flatMap(_.lastReplySinceCreationHrs).map(_.toDouble) val lastQuoteSinceCreationHrs = tweetFeatures.flatMap(_.lastQuoteSinceCreationHrs).map(_.toDouble) val timeSinceLastFavoriteHrs = getTimeSinceLastEngagementHrs(lastFavSinceCreationHrs, timeSinceSourceTweetCreationOpt) val timeSinceLastRetweetHrs = getTimeSinceLastEngagementHrs(lastRetweetSinceCreationHrs, timeSinceSourceTweetCreationOpt) val timeSinceLastReplyHrs = getTimeSinceLastEngagementHrs(lastReplySinceCreationHrs, timeSinceSourceTweetCreationOpt) val timeSinceLastQuoteHrs = getTimeSinceLastEngagementHrs(lastQuoteSinceCreationHrs, timeSinceSourceTweetCreationOpt) val nonPollingTimestampsMs = query.features.get.getOrElse(NonPollingTimesFeature, Seq.empty) val timeSinceLastNonPollingRequest = nonPollingTimestampsMs.headOption.map(query.queryTime.inMillis - _) val nonPollingRequestsSinceTweetCreation = if (nonPollingTimestampsMs.nonEmpty && timeSinceTweetCreationMs.isDefined) { nonPollingTimestampsMs .search(timeSinceTweetCreationMs.get)(Ordering[Long].reverse) .insertionPoint } else 0.0 val tweetAgeRatio = if (timeSinceTweetCreationMs.exists(_ > 0.0) && timeSinceLastNonPollingRequest.isDefined) { timeSinceLastNonPollingRequest.get / timeSinceTweetCreationMs.get.toDouble } else 0.0 val dataRecord = new DataRecord() .setFeatureValue(IS_TWEET_RECYCLED, false) .setFeatureValue(TWEET_AGE_RATIO, tweetAgeRatio) .setFeatureValueFromOption( TIME_SINCE_TWEET_CREATION, timeSinceTweetCreationMs.map(_.toDouble) ) .setFeatureValue( NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, nonPollingRequestsSinceTweetCreation ) .setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, lastFavSinceCreationHrs) .setFeatureValueFromOption(LAST_RETWEET_SINCE_CREATION_HRS, lastRetweetSinceCreationHrs) .setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, lastReplySinceCreationHrs) .setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, lastQuoteSinceCreationHrs) .setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, timeSinceLastFavoriteHrs) .setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, timeSinceLastRetweetHrs) .setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, timeSinceLastReplyHrs) .setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, timeSinceLastQuoteHrs) Stitch.value(FeatureMapBuilder().add(TweetTimeDataRecordFeature, dataRecord).build()) } private def getTimeSinceLastEngagementHrs( lastEngagementTimeSinceCreationHrsOpt: Option[Double], timeSinceTweetCreation: Option[Duration] ): Option[Double] = lastEngagementTimeSinceCreationHrsOpt.flatMap { lastEngagementTimeHrs => timeSinceTweetCreation.map(_.inHours - lastEngagementTimeHrs) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieContentFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.escherbird.{thriftscala => esb} import com.twitter.finagle.stats.Stat import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.MediaUnderstandingAnnotationIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieContentRepository import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content.ContentFeatureAdapter import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.home_mixer.util.tweetypie.content.FeatureExtractionHelper import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import com.twitter.timelines.prediction.common.util.MediaUnderstandingAnnotations import com.twitter.tweetypie.{thriftscala => tp} import com.twitter.util.Future import com.twitter.util.Return import com.twitter.util.Throw import com.twitter.util.Try import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object TweetypieContentDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TweetypieContentFeatureHydrator @Inject() ( @Named(TweetypieContentRepository) client: KeyValueRepository[Seq[Long], Long, tp.Tweet], override val statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetypieContent") override val features: Set[Feature[_, _]] = Set( MediaUnderstandingAnnotationIdsFeature, TweetypieContentDataRecordFeature ) override val statScope: String = identifier.toString private val bulkRequestLatencyStat = statsReceiver.scope(statScope).scope("bulkRequest").stat("latency_ms") private val postTransformerLatencyStat = statsReceiver.scope(statScope).scope("postTransformer").stat("latency_ms") private val bulkPostTransformerLatencyStat = statsReceiver.scope(statScope).scope("bulkPostTransformer").stat("latency_ms") private val DefaultDataRecord: DataRecord = new DataRecord() override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val tweetIdsToHydrate = candidates.map(getCandidateOriginalTweetId).distinct val response: Future[KeyValueResult[Long, tp.Tweet]] = Stat.timeFuture(bulkRequestLatencyStat) { if (tweetIdsToHydrate.isEmpty) Future.value(KeyValueResult.empty) else client(tweetIdsToHydrate) } response.flatMap { result => Stat.timeFuture(bulkPostTransformerLatencyStat) { OffloadFuturePools .parallelize[CandidateWithFeatures[TweetCandidate], Try[(Seq[Long], DataRecord)]]( candidates, parTransformer(result, _), parallelism = 32, default = Return((Seq.empty, DefaultDataRecord)) ).map { _.map { case Return(result) => FeatureMapBuilder() .add(MediaUnderstandingAnnotationIdsFeature, result._1) .add(TweetypieContentDataRecordFeature, result._2) .build() case Throw(e) => FeatureMapBuilder() .add(MediaUnderstandingAnnotationIdsFeature, Throw(e)) .add(TweetypieContentDataRecordFeature, Throw(e)) .build() } } } } } private def parTransformer( result: KeyValueResult[Long, tp.Tweet], candidate: CandidateWithFeatures[TweetCandidate] ): Try[(Seq[Long], DataRecord)] = { val originalTweetId = Some(getCandidateOriginalTweetId(candidate)) val value = observedGet(key = originalTweetId, keyValueResult = result) Stat.time(postTransformerLatencyStat)(postTransformer(value)) } private def postTransformer( result: Try[Option[tp.Tweet]] ): Try[(Seq[Long], DataRecord)] = { result.map { tweet => val transformedValue = tweet.map(FeatureExtractionHelper.extractFeatures) val semanticAnnotations = transformedValue .flatMap { contentFeatures => contentFeatures.semanticCoreAnnotations.map { getNonSensitiveHighRecallMediaUnderstandingAnnotationEntityIds } }.getOrElse(Seq.empty) val dataRecord = ContentFeatureAdapter.adaptToDataRecords(transformedValue).asScala.head (semanticAnnotations, dataRecord) } } private def getCandidateOriginalTweetId( candidate: CandidateWithFeatures[TweetCandidate] ): Long = { candidate.features .getOrElse(SourceTweetIdFeature, None).getOrElse(candidate.candidate.id) } private def getNonSensitiveHighRecallMediaUnderstandingAnnotationEntityIds( semanticCoreAnnotations: Seq[esb.TweetEntityAnnotation] ): Seq[Long] = semanticCoreAnnotations .filter(MediaUnderstandingAnnotations.isEligibleNonSensitiveHighRecallMUAnnotation) .map(_.entityId) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieStaticEntitiesFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.google.inject.name.Named import com.twitter.conversions.DurationOps.RichDuration import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieStaticEntitiesCache import com.twitter.home_mixer.util.tweetypie.RequestFields import com.twitter.home_mixer.util.tweetypie.content.TweetMediaFeaturesExtractor import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.TtlCache import com.twitter.spam.rtf.{thriftscala => sp} import com.twitter.stitch.Stitch import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} import com.twitter.tweetypie.{thriftscala => tp} import javax.inject.Inject import javax.inject.Singleton @Singleton class TweetypieStaticEntitiesFeatureHydrator @Inject() ( tweetypieStitchClient: TweetypieStitchClient, @Named(TweetypieStaticEntitiesCache) cacheClient: TtlCache[Long, tp.Tweet]) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetypieStaticEntities") override val features: Set[Feature[_, _]] = Set( AuthorIdFeature, DirectedAtUserIdFeature, ExclusiveConversationAuthorIdFeature, HasImageFeature, HasVideoFeature, InReplyToTweetIdFeature, InReplyToUserIdFeature, IsRetweetFeature, MentionScreenNameFeature, MentionUserIdFeature, QuotedTweetIdFeature, QuotedUserIdFeature, SourceTweetIdFeature, SourceUserIdFeature ) private val CacheTTL = 24.hours private val DefaultFeatureMap = FeatureMapBuilder() .add(AuthorIdFeature, None) .add(DirectedAtUserIdFeature, None) .add(ExclusiveConversationAuthorIdFeature, None) .add(HasImageFeature, false) .add(HasVideoFeature, false) .add(InReplyToTweetIdFeature, None) .add(InReplyToUserIdFeature, None) .add(IsRetweetFeature, false) .add(MentionScreenNameFeature, Seq.empty) .add(MentionUserIdFeature, Seq.empty) .add(QuotedTweetIdFeature, None) .add(QuotedUserIdFeature, None) .add(SourceTweetIdFeature, None) .add(SourceUserIdFeature, None) .build() /** * Steps: * 1. query cache with all candidates * 2. create a cached feature map * 3. iterate candidates to hydrate features * 3.a transform cached candidates * 3.b hydrate non-cached candidates from Tweetypie and write to cache */ override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { val tweetIds = candidates.map(_.candidate.id) val cachedTweetsMapFu = cacheClient .get(tweetIds) .map(_.found) Stitch.callFuture(cachedTweetsMapFu).flatMap { cachedTweets => Stitch.collect { candidates.map { candidate => if (cachedTweets.contains(candidate.candidate.id)) Stitch.value(createFeatureMap(cachedTweets(candidate.candidate.id))) else readFromTweetypie(query, candidate) } } } } private def createFeatureMap(tweet: tp.Tweet): FeatureMap = { val coreData = tweet.coreData val quotedTweet = tweet.quotedTweet val mentions = tweet.mentions.getOrElse(Seq.empty) val share = coreData.flatMap(_.share) val reply = coreData.flatMap(_.reply) FeatureMapBuilder() .add(AuthorIdFeature, coreData.map(_.userId)) .add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId))) .add( ExclusiveConversationAuthorIdFeature, tweet.exclusiveTweetControl.map(_.conversationAuthorId)) .add(HasImageFeature, TweetMediaFeaturesExtractor.hasImage(tweet)) .add(HasVideoFeature, TweetMediaFeaturesExtractor.hasVideo(tweet)) .add(InReplyToTweetIdFeature, reply.flatMap(_.inReplyToStatusId)) .add(InReplyToUserIdFeature, reply.map(_.inReplyToUserId)) .add(IsRetweetFeature, share.isDefined) .add(MentionScreenNameFeature, mentions.map(_.screenName)) .add(MentionUserIdFeature, mentions.flatMap(_.userId)) .add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId)) .add(QuotedUserIdFeature, quotedTweet.map(_.userId)) .add(SourceTweetIdFeature, share.map(_.sourceStatusId)) .add(SourceUserIdFeature, share.map(_.sourceUserId)) .build() } private def readFromTweetypie( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate] ): Stitch[FeatureMap] = { tweetypieStitchClient .getTweetFields( tweetId = candidate.candidate.id, options = tp.GetTweetFieldsOptions( tweetIncludes = RequestFields.TweetStaticEntitiesFields, includeRetweetedTweet = false, includeQuotedTweet = false, forUserId = query.getOptionalUserId, // Needed to get protected Tweets for certain users visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible, safetyLevel = Some(sp.SafetyLevel.FilterNone) // VF is handled in the For You product ) ).map { case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), _, _) => cacheClient.set(candidate.candidate.id, found.tweet, CacheTTL) createFeatureMap(found.tweet) case _ => DefaultFeatureMap + (AuthorIdFeature, candidate.features.getOrElse(AuthorIdFeature, None)) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieVisibilityFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature import com.twitter.home_mixer.model.HomeFeatures.OonNsfwFeature import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnableTweetEntityServiceVisibilityMigrationParam import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithLongTimeout import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.spam.rtf.{thriftscala => rtf} import com.twitter.stitch.Stitch import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} import com.twitter.strato.client.Client import com.twitter.strato.generated.client.tweetypie.managed.HomeMixerOnTweetClientColumn import com.twitter.tweetypie.{thriftscala => tp} import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton case class TweetypieVisibilityFeatures( communityId: Option[Long], isHydrated: Boolean, isNsfw: Boolean, oonNsfw: Boolean, tweetText: Option[String], tweetLanguage: Option[String], visibilityReason: Option[rtf.FilteredReason]) @Singleton class TweetypieVisibilityFeatureHydrator @Inject() ( tweetypieStitchClient: TweetypieStitchClient, statsReceiver: StatsReceiver, @Named(BatchedStratoClientWithLongTimeout) stratoClient: Client) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Logging { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetypieVisibility") private val tweetypieTweetsVisibilityFoundCounter = statsReceiver.counter("VisibilityFeatureTweetypieTweetsFound") private val tweetypieTweetsVisibilityNotFoundCounter = statsReceiver.counter("VisibilityFeatureTweetypieTweetsNotFound") private val tesTweetsVisibilityFoundCounter = statsReceiver.counter("VisibilityFeatureTesTweetsFound") private val tesTweetsVisibilityNotFoundCounter = statsReceiver.counter("VisibilityFeatureTesTweetsNotFound") override val features: Set[Feature[_, _]] = Set( CommunityIdFeature, IsHydratedFeature, IsNsfw, OonNsfwFeature, TweetTextFeature, TweetLanguageFeature, VisibilityReason ) private val HydrationFields: Set[tp.TweetInclude] = Set( tp.TweetInclude.TweetFieldId(tp.Tweet.CommunitiesField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.CoreDataField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.IdField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.LanguageField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.QuotedTweetField.id) ) private val DefaultTweetFieldsOptions = tp.GetTweetFieldsOptions( tweetIncludes = HydrationFields, includeRetweetedTweet = true, includeQuotedTweet = true, visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible, safetyLevel = Some(rtf.SafetyLevel.TimelineHome) ) private val OutOfNetworkTweetFieldsOptions = DefaultTweetFieldsOptions.copy(safetyLevel = Some(rtf.SafetyLevel.TimelineHomeRecommendations)) private val DefaultTweetypieVisibilityFeatures = TweetypieVisibilityFeatures( communityId = None, isHydrated = false, isNsfw = false, oonNsfw = false, tweetLanguage = None, tweetText = None, visibilityReason = None ) private def buildFeatureMap( gtfResult: Stitch[tp.GetTweetFieldsResult], inNetwork: Boolean, fromTes: Boolean, tweetId: Long ): Stitch[(Long, TweetypieVisibilityFeatures)] = { gtfResult.map { case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quote, _) => if (fromTes) tesTweetsVisibilityFoundCounter.incr() else tweetypieTweetsVisibilityFoundCounter.incr() val coreData = found.tweet.coreData val isNsfwAdmin = coreData.exists(_.nsfwAdmin) val isNsfwUser = coreData.exists(_.nsfwUser) val sourceTweetIsNsfw = found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser)) val quotedTweetDropped = quote.exists { case _: tp.TweetFieldsResultState.Filtered => true case _: tp.TweetFieldsResultState.NotFound => true case _ => false } val quotedTweetIsNsfw = quote.exists { case quoteTweet: tp.TweetFieldsResultState.Found => quoteTweet.found.tweet.coreData.exists(data => data.nsfwAdmin || data.nsfwUser) case _ => false } val isNsfw = isNsfwAdmin || isNsfwUser || sourceTweetIsNsfw || quotedTweetIsNsfw val communityId = found.tweet.communities.flatMap(_.communityIds.headOption) tweetId -> TweetypieVisibilityFeatures( communityId = communityId, // Since this tweet was Found, it is not dropped, so we only need to check if there // was a dropped quoted tweet. isHydrated = !quotedTweetDropped, isNsfw = isNsfw, oonNsfw = !inNetwork && isNsfw, tweetLanguage = found.tweet.language.map(_.language), tweetText = coreData.map(_.text), visibilityReason = found.suppressReason ) case _ => if (fromTes) tesTweetsVisibilityNotFoundCounter.incr() else tweetypieTweetsVisibilityNotFoundCounter.incr() tweetId -> DefaultTweetypieVisibilityFeatures } } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { try { val followedUserIds = query.features.get.getOrElse(SGSFollowedUsersFeature, Seq.empty).toSet val inNetworkTweetFieldsOptions = DefaultTweetFieldsOptions.copy(forUserId = Some(query.getRequiredUserId)) val outOfNetworkTweetFieldsOptions = OutOfNetworkTweetFieldsOptions.copy(forUserId = Some(query.getRequiredUserId)) val resultsStitch = Stitch .collect { candidates .flatMap { candidate => val ancestors = candidate.features.getOrElse(AncestorsFeature, Seq.empty).map { ancestor => val inNetwork = ancestor.userId == query.getRequiredUserId || followedUserIds.contains(ancestor.userId) (ancestor.tweetId, ancestor.userId, inNetwork) } val authorId = candidate.features.get(AuthorIdFeature).get val inNetwork = authorId == query.getRequiredUserId || followedUserIds.contains(authorId) Seq((candidate.candidate.id, authorId, inNetwork)) ++ ancestors.headOption ++ ancestors.lastOption }.distinct.map { case (tweetId, _, inNetwork) => val gtfOptions = if (inNetwork) inNetworkTweetFieldsOptions else outOfNetworkTweetFieldsOptions try { if (query.params(EnableTweetEntityServiceVisibilityMigrationParam)) { val fetcher = new HomeMixerOnTweetClientColumn(stratoClient).fetcher val response = fetcher.fetch(tweetId, gtfOptions).map(_.v) response.flatMap { case Some(result) => buildFeatureMap(Stitch.value(result), inNetwork, fromTes = true, tweetId) case None => tesTweetsVisibilityNotFoundCounter.incr() Stitch.value(tweetId -> DefaultTweetypieVisibilityFeatures) } } else { buildFeatureMap( tweetypieStitchClient.getTweetFields(tweetId, gtfOptions), inNetwork, fromTes = false, tweetId ) } } catch { case e: Exception => error(s"2 - Error fetching tweetypie visibility: $e") Stitch.value(tweetId -> DefaultTweetypieVisibilityFeatures) } } }.onFailure { case e: Exception => error(s"1 - Error fetching tweetypie visibility: $e") } resultsStitch.map { results => val resultsMap = results.toMap candidates.map { candidate => val ancestors = candidate.features.getOrElse(AncestorsFeature, Seq.empty) val ancestorTweetypieVisibilityFeatures = (ancestors.headOption ++ ancestors.lastOption).toSeq.distinct.map { ancestor => resultsMap.getOrElse(ancestor.tweetId, DefaultTweetypieVisibilityFeatures) } val ancestorsHydrated = ancestorTweetypieVisibilityFeatures.map(_.isHydrated).forall(identity) val ancestorsOonNsfw = ancestorTweetypieVisibilityFeatures.map(_.oonNsfw).exists(identity) val ancestorsNsfw = ancestorTweetypieVisibilityFeatures.map(_.isNsfw).exists(identity) val tweetypieVisibilityFeatures = resultsMap.getOrElse(candidate.candidate.id, DefaultTweetypieVisibilityFeatures) FeatureMapBuilder() .add(CommunityIdFeature, tweetypieVisibilityFeatures.communityId) .add(IsHydratedFeature, tweetypieVisibilityFeatures.isHydrated && ancestorsHydrated) .add(IsNsfw, Some(tweetypieVisibilityFeatures.isNsfw || ancestorsNsfw)) .add(OonNsfwFeature, tweetypieVisibilityFeatures.oonNsfw || ancestorsOonNsfw) .add(TweetLanguageFeature, tweetypieVisibilityFeatures.tweetLanguage) .add(TweetTextFeature, tweetypieVisibilityFeatures.tweetText) .add(VisibilityReason, tweetypieVisibilityFeatures.visibilityReason) .build() } } } catch { case e: Exception => error(s"3 - Error fetching tweetypie visibility: $e") Stitch.value(Seq.empty) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinAuthorFollowFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollowFeatureRepository import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinAuthorFollowEmbeddingsAdapter import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.ml.api.DataRecord import com.twitter.ml.api.{thriftscala => ml} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.repository.KeyValueRepository import com.twitter.servo.repository.KeyValueResult import com.twitter.stitch.Stitch import com.twitter.util.Future import com.twitter.util.Try import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object TwhinAuthorFollowFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TwhinAuthorFollowFeatureHydrator @Inject() ( @Named(TwhinAuthorFollowFeatureRepository) client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], override val statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TwhinAuthorFollow") override val features: Set[Feature[_, _]] = Set(TwhinAuthorFollowFeature) override val statScope: String = identifier.toString override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val possiblyAuthorIds = extractKeys(candidates) val authorIds = possiblyAuthorIds.flatten val response: Future[KeyValueResult[Long, ml.FloatTensor]] = if (authorIds.isEmpty) Future.value(KeyValueResult.empty) else client(authorIds) response.map { result => possiblyAuthorIds.map { possiblyAuthorId => val value = observedGet(key = possiblyAuthorId, keyValueResult = result) val transformedValue = postTransformer(value) FeatureMapBuilder().add(TwhinAuthorFollowFeature, transformedValue).build() } } } private def postTransformer(embedding: Try[Option[ml.FloatTensor]]): Try[DataRecord] = { embedding.map { floatTensor => TwhinAuthorFollowEmbeddingsAdapter.adaptToDataRecords(floatTensor).asScala.head } } private def extractKeys( candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[Long]] = { candidates.map { candidate => CandidatesUtil.getOriginalAuthorId(candidate.features) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserEngagementFeatureRepository import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinUserEngagementEmbeddingsAdapter import com.twitter.ml.api.DataRecord import com.twitter.ml.api.{thriftscala => ml} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import com.twitter.util.Return import com.twitter.util.Throw import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object TwhinUserEngagementFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TwhinUserEngagementQueryFeatureHydrator @Inject() ( @Named(TwhinUserEngagementFeatureRepository) client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TwhinUserEngagement") override val features: Set[Feature[_, _]] = Set(TwhinUserEngagementFeature) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyLossCounter = scopedStatsReceiver.counter("key/loss") private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId Stitch.callFuture(client(Seq(userId))).map { results => val embedding: Option[ml.FloatTensor] = results(userId) match { case Return(value) => if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() else keyLossCounter.incr() value case Throw(_) => keyFailureCounter.incr() None case _ => None } val dataRecord = TwhinUserEngagementEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head FeatureMapBuilder() .add(TwhinUserEngagementFeature, dataRecord) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserFollowFeatureRepository import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinUserFollowEmbeddingsAdapter import com.twitter.ml.api.DataRecord import com.twitter.ml.api.{thriftscala => ml} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import com.twitter.util.Return import com.twitter.util.Throw import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ object TwhinUserFollowFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TwhinUserFollowQueryFeatureHydrator @Inject() ( @Named(TwhinUserFollowFeatureRepository) client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TwhinUserFollow") override val features: Set[Feature[_, _]] = Set(TwhinUserFollowFeature) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val keyFoundCounter = scopedStatsReceiver.counter("key/found") private val keyLossCounter = scopedStatsReceiver.counter("key/loss") private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val userId = query.getRequiredUserId Stitch.callFuture(client(Seq(userId))).map { results => val embedding: Option[ml.FloatTensor] = results(userId) match { case Return(value) => if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() else keyLossCounter.incr() value case Throw(_) => keyFailureCounter.incr() None case _ => None } val dataRecord = TwhinUserFollowEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head FeatureMapBuilder() .add(TwhinUserFollowFeature, dataRecord) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserFollowedTopicIdsFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserFollowedTopicIdsRepository import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import com.twitter.util.Future import com.twitter.util.Try import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton object UserFollowedTopicIdsFeature extends Feature[TweetCandidate, Seq[Long]] @Singleton class UserFollowedTopicIdsFeatureHydrator @Inject() ( @Named(UserFollowedTopicIdsRepository) client: KeyValueRepository[Seq[Long], Long, Seq[Long]], override val statsReceiver: StatsReceiver) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserFollowedTopicIds") override val features: Set[Feature[_, _]] = Set(UserFollowedTopicIdsFeature) override val statScope: String = identifier.toString override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val possiblyAuthorIds = extractKeys(candidates) val authorIds = possiblyAuthorIds.flatten val response: Future[KeyValueResult[Long, Seq[Long]]] = if (authorIds.isEmpty) Future.value(KeyValueResult.empty) else client(authorIds) response.map { result => possiblyAuthorIds.map { possiblyAuthorId => val value = observedGet(key = possiblyAuthorId, keyValueResult = result) val transformedValue = postTransformer(value) FeatureMapBuilder().add(UserFollowedTopicIdsFeature, transformedValue).build() } } } private def postTransformer(input: Try[Option[Seq[Long]]]): Try[Seq[Long]] = { input.map(_.getOrElse(Seq.empty[Long])) } private def extractKeys( candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[Long]] = { candidates.map { candidate => candidate.features .getTry(AuthorIdFeature) .toOption .flatten } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserLanguagesFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserLanguagesRepository import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.search.common.constants.{thriftscala => scc} import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton object UserLanguagesFeature extends Feature[PipelineQuery, Seq[scc.ThriftLanguage]] @Singleton case class UserLanguagesFeatureHydrator @Inject() ( @Named(UserLanguagesRepository) client: KeyValueRepository[Seq[Long], Long, Seq[ scc.ThriftLanguage ]], statsReceiver: StatsReceiver) extends QueryFeatureHydrator[PipelineQuery] with ObservedKeyValueResultHandler { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserLanguages") override val features: Set[Feature[_, _]] = Set(UserLanguagesFeature) override val statScope: String = identifier.toString override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val key = query.getRequiredUserId Stitch.callFuture(client(Seq(key))).map { result => val feature = observedGet(key = Some(key), keyValueResult = result).map(_.getOrElse(Seq.empty)) FeatureMapBuilder() .add(UserLanguagesFeature, feature) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserStateQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.user_health.v1.{thriftscala => uhv1} import com.twitter.timelines.user_health.{thriftscala => uh} import com.twitter.user_session_store.ReadOnlyUserSessionStore import com.twitter.user_session_store.ReadRequest import com.twitter.user_session_store.UserSessionDataset import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset import javax.inject.Inject import javax.inject.Singleton @Singleton case class UserStateQueryFeatureHydrator @Inject() ( userSessionStore: ReadOnlyUserSessionStore) extends QueryFeatureHydrator[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserState") override val features: Set[Feature[_, _]] = Set(UserStateFeature) private val datasets: Set[UserSessionDataset] = Set(UserSessionDataset.UserHealth) override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { userSessionStore .read(ReadRequest(query.getRequiredUserId, datasets)) .map { userSession => val userState = userSession.flatMap { _.userHealth match { case Some(uh.UserHealth.V1(uhv1.UserHealth(userState))) => userState case _ => None } } FeatureMapBuilder() .add(UserStateFeature, userState) .build() } } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UtegFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.FavoritedByCountFeature import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.home_mixer.model.HomeFeatures.RepliedByCountFeature import com.twitter.home_mixer.model.HomeFeatures.RepliedByEngagerIdsFeature import com.twitter.home_mixer.model.HomeFeatures.RetweetedByCountFeature import com.twitter.home_mixer.model.HomeFeatures.RetweetedByEngagerIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.UtegSocialProofRepository import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.Conditionally import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.recos.recos_common.{thriftscala => rc} import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.servo.repository.KeyValueRepository import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class UtegFeatureHydrator @Inject() ( @Named(UtegSocialProofRepository) client: KeyValueRepository[ (Seq[Long], (Long, Map[Long, Double])), Long, uteg.TweetRecommendation ]) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with Conditionally[PipelineQuery] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Uteg") override val features: Set[Feature[_, _]] = Set( FavoritedByUserIdsFeature, RetweetedByEngagerIdsFeature, RepliedByEngagerIdsFeature, FavoritedByCountFeature, RetweetedByCountFeature, RepliedByCountFeature ) override def onlyIf(query: PipelineQuery): Boolean = query.features .exists(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double]).nonEmpty) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val seedUserWeights = query.features.map(_.get(RealGraphInNetworkScoresFeature)).get val sourceTweetIds = candidates.flatMap(_.features.getOrElse(SourceTweetIdFeature, None)) val inReplyToTweetIds = candidates.flatMap(_.features.getOrElse(InReplyToTweetIdFeature, None)) val tweetIds = candidates.map(_.candidate.id) val tweetIdsToSend = (tweetIds ++ sourceTweetIds ++ inReplyToTweetIds).distinct val utegQuery = (tweetIdsToSend, (query.getRequiredUserId, seedUserWeights)) client(utegQuery).map(handleResponse(candidates, _)) } private def handleResponse( candidates: Seq[CandidateWithFeatures[TweetCandidate]], results: KeyValueResult[Long, uteg.TweetRecommendation], ): Seq[FeatureMap] = { candidates.map { candidate => val inNetwork = candidate.features.getOrElse(FromInNetworkSourceFeature, false) val candidateProof = results(candidate.candidate.id).toOption.flatten val sourceProof = candidate.features .getOrElse(SourceTweetIdFeature, None).flatMap(results(_).toOption.flatten) val proofs = Seq(candidateProof, sourceProof).flatten.map(_.socialProofByType) val favoritedBy = proofs.flatMap(_.get(rc.SocialProofType.Favorite)).flatten val retweetedBy = proofs.flatMap(_.get(rc.SocialProofType.Retweet)).flatten val repliedBy = proofs.flatMap(_.get(rc.SocialProofType.Reply)).flatten val (favoritedByCount, retweetedByCount, repliedByCount) = if (!inNetwork) { (favoritedBy.size.toDouble, retweetedBy.size.toDouble, repliedBy.size.toDouble) } else { (0.0, 0.0, 0.0) } FeatureMapBuilder() .add(FavoritedByUserIdsFeature, favoritedBy) .add(RetweetedByEngagerIdsFeature, retweetedBy) .add(RepliedByEngagerIdsFeature, repliedBy) .add(FavoritedByCountFeature, favoritedByCount) .add(RetweetedByCountFeature, retweetedByCount) .add(RepliedByCountFeature, repliedByCount) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ValidLikedByUserIdsFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ValidLikedByUserIdsFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object ValidLikedByUserIdsFeatureHydrator extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ValidLikedByUserIds") override val features: Set[Feature[_, _]] = Set(ValidLikedByUserIdsFeature) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { Stitch.value { candidates.map { candidate => val validLikers = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) FeatureMapBuilder().add(ValidLikedByUserIdsFeature, validLikers).build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.author_features import com.twitter.home_mixer.util.DataRecordUtil import com.twitter.ml.api.DataRecord import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.util.CompactDataRecordConverter import com.twitter.ml.api.util.FDsl._ import com.twitter.timelines.author_features.v1.{thriftjava => af} import com.twitter.timelines.prediction.common.adapters.TimelinesAdapterBase import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures import scala.collection.JavaConverters._ object AuthorFeaturesAdapter extends TimelinesAdapterBase[af.AuthorFeatures] { private val Prefix = "original_author.timelines.original_author_aggregates." private val typedAggregateGroups = TimelinesAggregationConfig.originalAuthorAggregatesV1.buildTypedAggregateGroups() private val aggregateFeaturesRenameMap: Map[Feature[_], Feature[_]] = typedAggregateGroups.map(_.outputFeaturesToRenamedOutputFeatures(Prefix)).reduce(_ ++ _) private val prefixedOriginalAuthorAggregateFeatures = typedAggregateGroups.flatMap(_.allOutputFeatures).map { feature => aggregateFeaturesRenameMap.getOrElse(feature, feature) } private val authorFeatures = prefixedOriginalAuthorAggregateFeatures ++ Seq( UserHealthFeatures.AuthorState, UserHealthFeatures.NumAuthorFollowers, UserHealthFeatures.NumAuthorConnectDays, UserHealthFeatures.NumAuthorConnect ) private val aggregateFeatureContext: FeatureContext = new FeatureContext(typedAggregateGroups.flatMap(_.allOutputFeatures).asJava) private lazy val prefixedAggregateFeatureContext: FeatureContext = new FeatureContext(prefixedOriginalAuthorAggregateFeatures.asJava) override val getFeatureContext: FeatureContext = new FeatureContext(authorFeatures: _*) override val commonFeatures: Set[Feature[_]] = Set.empty private val compactDataRecordConverter = new CompactDataRecordConverter() override def adaptToDataRecords( authorFeatures: af.AuthorFeatures ): java.util.List[DataRecord] = { val dataRecord = if (authorFeatures.aggregates != null) { val originalAuthorAggregatesDataRecord = compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates) DataRecordUtil.applyRename( originalAuthorAggregatesDataRecord, aggregateFeatureContext, prefixedAggregateFeatureContext, aggregateFeaturesRenameMap) } else new DataRecord if (authorFeatures.user_health != null) { val userHealth = authorFeatures.user_health if (userHealth.user_state != null) { dataRecord.setFeatureValue( UserHealthFeatures.AuthorState, userHealth.user_state.getValue.toLong ) } dataRecord.setFeatureValue( UserHealthFeatures.NumAuthorFollowers, userHealth.num_followers.toDouble ) dataRecord.setFeatureValue( UserHealthFeatures.NumAuthorConnectDays, userHealth.num_connect_days.toDouble ) dataRecord.setFeatureValue( UserHealthFeatures.NumAuthorConnect, userHealth.num_connect.toDouble ) } List(dataRecord).asJava } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "src/java/com/twitter/ml/api:api-base", "src/java/com/twitter/ml/api/util", "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/scala/com/twitter/timelines/prediction/common/aggregates", "src/scala/com/twitter/timelines/prediction/features/user_health", "src/thrift/com/twitter/ml/api:data-java", "src/thrift/com/twitter/timelines/author_features:thrift-java", "timelines/data_processing/ml_util/aggregation_framework", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/timelines/prediction/common/adapters", "src/scala/com/twitter/timelines/prediction/features/common", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/ContentFeatureAdapter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content import com.twitter.home_mixer.model.ContentFeatures import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.ml.api.util.DataRecordConverters._ import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.common.adapters.TweetLengthType import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import com.twitter.timelines.prediction.features.conversation_features.ConversationFeatures import scala.collection.JavaConverters._ object ContentFeatureAdapter extends TimelinesMutatingAdapterBase[Option[ContentFeatures]] { override val getFeatureContext: FeatureContext = new FeatureContext( ConversationFeatures.IS_SELF_THREAD_TWEET, ConversationFeatures.IS_LEAF_IN_SELF_THREAD, TimelinesSharedFeatures.ASPECT_RATIO_DEN, TimelinesSharedFeatures.ASPECT_RATIO_NUM, TimelinesSharedFeatures.BIT_RATE, TimelinesSharedFeatures.CLASSIFICATION_LABELS, TimelinesSharedFeatures.COLOR_1_BLUE, TimelinesSharedFeatures.COLOR_1_GREEN, TimelinesSharedFeatures.COLOR_1_PERCENTAGE, TimelinesSharedFeatures.COLOR_1_RED, TimelinesSharedFeatures.FACE_AREAS, TimelinesSharedFeatures.HAS_APP_INSTALL_CALL_TO_ACTION, TimelinesSharedFeatures.HAS_DESCRIPTION, TimelinesSharedFeatures.HAS_QUESTION, TimelinesSharedFeatures.HAS_SELECTED_PREVIEW_IMAGE, TimelinesSharedFeatures.HAS_TITLE, TimelinesSharedFeatures.HAS_VISIT_SITE_CALL_TO_ACTION, TimelinesSharedFeatures.HAS_WATCH_NOW_CALL_TO_ACTION, TimelinesSharedFeatures.HEIGHT_1, TimelinesSharedFeatures.HEIGHT_2, TimelinesSharedFeatures.HEIGHT_3, TimelinesSharedFeatures.HEIGHT_4, TimelinesSharedFeatures.IS_360, TimelinesSharedFeatures.IS_EMBEDDABLE, TimelinesSharedFeatures.IS_MANAGED, TimelinesSharedFeatures.IS_MONETIZABLE, TimelinesSharedFeatures.MEDIA_PROVIDERS, TimelinesSharedFeatures.NUM_CAPS, TimelinesSharedFeatures.NUM_COLOR_PALLETTE_ITEMS, TimelinesSharedFeatures.NUM_FACES, TimelinesSharedFeatures.NUM_MEDIA_TAGS, TimelinesSharedFeatures.NUM_NEWLINES, TimelinesSharedFeatures.NUM_STICKERS, TimelinesSharedFeatures.NUM_WHITESPACES, TimelinesSharedFeatures.RESIZE_METHOD_1, TimelinesSharedFeatures.RESIZE_METHOD_2, TimelinesSharedFeatures.RESIZE_METHOD_3, TimelinesSharedFeatures.RESIZE_METHOD_4, TimelinesSharedFeatures.TWEET_LENGTH, TimelinesSharedFeatures.TWEET_LENGTH_TYPE, TimelinesSharedFeatures.VIDEO_DURATION, TimelinesSharedFeatures.VIEW_COUNT, TimelinesSharedFeatures.WIDTH_1, TimelinesSharedFeatures.WIDTH_2, TimelinesSharedFeatures.WIDTH_3, TimelinesSharedFeatures.WIDTH_4, ) override val commonFeatures: Set[Feature[_]] = Set.empty private def getTweetLengthType(tweetLength: Int): Long = { tweetLength match { case x if 0 > x || 280 < x => TweetLengthType.INVALID case x if 0 <= x && x <= 30 => TweetLengthType.VERY_SHORT case x if 30 < x && x <= 60 => TweetLengthType.SHORT case x if 60 < x && x <= 90 => TweetLengthType.MEDIUM case x if 90 < x && x <= 140 => TweetLengthType.LENGTHY case x if 140 < x && x <= 210 => TweetLengthType.VERY_LENGTHY case x if x > 210 => TweetLengthType.MAXIMUM_LENGTH } } override def setFeatures( contentFeatures: Option[ContentFeatures], richDataRecord: RichDataRecord ): Unit = { if (contentFeatures.nonEmpty) { val features = contentFeatures.get // Conversation Features richDataRecord.setFeatureValueFromOption( ConversationFeatures.IS_SELF_THREAD_TWEET, Some(features.selfThreadMetadata.nonEmpty) ) richDataRecord.setFeatureValueFromOption( ConversationFeatures.IS_LEAF_IN_SELF_THREAD, features.selfThreadMetadata.map(_.isLeaf) ) // Media Features richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.ASPECT_RATIO_DEN, features.aspectRatioDen.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.ASPECT_RATIO_NUM, features.aspectRatioNum.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.BIT_RATE, features.bitRate.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HEIGHT_1, features.heights.flatMap(_.lift(0)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HEIGHT_2, features.heights.flatMap(_.lift(1)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HEIGHT_3, features.heights.flatMap(_.lift(2)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HEIGHT_4, features.heights.flatMap(_.lift(3)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.NUM_MEDIA_TAGS, features.numMediaTags.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.RESIZE_METHOD_1, features.resizeMethods.flatMap(_.lift(0)).map(_.toLong) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.RESIZE_METHOD_2, features.resizeMethods.flatMap(_.lift(1)).map(_.toLong) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.RESIZE_METHOD_3, features.resizeMethods.flatMap(_.lift(2)).map(_.toLong) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.RESIZE_METHOD_4, features.resizeMethods.flatMap(_.lift(3)).map(_.toLong) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.VIDEO_DURATION, features.videoDurationMs.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.WIDTH_1, features.widths.flatMap(_.lift(0)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.WIDTH_2, features.widths.flatMap(_.lift(1)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.WIDTH_3, features.widths.flatMap(_.lift(2)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.WIDTH_4, features.widths.flatMap(_.lift(3)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.NUM_COLOR_PALLETTE_ITEMS, features.numColors.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.COLOR_1_RED, features.dominantColorRed.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.COLOR_1_BLUE, features.dominantColorBlue.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.COLOR_1_GREEN, features.dominantColorGreen.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.COLOR_1_PERCENTAGE, features.dominantColorPercentage ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.MEDIA_PROVIDERS, features.mediaOriginProviders.map(_.toSet.asJava) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.IS_360, features.is360 ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.VIEW_COUNT, features.viewCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.IS_MANAGED, features.isManaged ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.IS_MONETIZABLE, features.isMonetizable ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.IS_EMBEDDABLE, features.isEmbeddable ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.NUM_STICKERS, features.stickerIds.map(_.length.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.NUM_FACES, features.faceAreas.map(_.length.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.FACE_AREAS, // guard for exception from max on empty seq features.faceAreas.map(faceAreas => faceAreas.map(_.toDouble).reduceOption(_ max _).getOrElse(0.0)) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HAS_SELECTED_PREVIEW_IMAGE, features.hasSelectedPreviewImage ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HAS_TITLE, features.hasTitle ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HAS_DESCRIPTION, features.hasDescription ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HAS_VISIT_SITE_CALL_TO_ACTION, features.hasVisitSiteCallToAction ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HAS_APP_INSTALL_CALL_TO_ACTION, features.hasAppInstallCallToAction ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HAS_WATCH_NOW_CALL_TO_ACTION, features.hasWatchNowCallToAction ) // text features richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.NUM_CAPS, Some(features.numCaps.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.TWEET_LENGTH, Some(features.length.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.TWEET_LENGTH_TYPE, Some(getTweetLengthType(features.length.toInt)) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.NUM_WHITESPACES, Some(features.numWhiteSpaces.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HAS_QUESTION, Some(features.hasQuestion) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.NUM_NEWLINES, features.numNewlines.map(_.toDouble) ) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/InReplyToContentFeatureAdapter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content import com.twitter.home_mixer.model.ContentFeatures import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.ml.api.util.DataRecordConverters.RichDataRecordWrapper import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.features.common.InReplyToTweetTimelinesSharedFeatures object InReplyToContentFeatureAdapter extends TimelinesMutatingAdapterBase[Option[ContentFeatures]] { override val getFeatureContext: FeatureContext = new FeatureContext( // Media Features InReplyToTweetTimelinesSharedFeatures.ASPECT_RATIO_DEN, InReplyToTweetTimelinesSharedFeatures.ASPECT_RATIO_NUM, InReplyToTweetTimelinesSharedFeatures.HEIGHT_1, InReplyToTweetTimelinesSharedFeatures.HEIGHT_2, InReplyToTweetTimelinesSharedFeatures.VIDEO_DURATION, // TextFeatures InReplyToTweetTimelinesSharedFeatures.NUM_CAPS, InReplyToTweetTimelinesSharedFeatures.TWEET_LENGTH, InReplyToTweetTimelinesSharedFeatures.HAS_QUESTION, ) override val commonFeatures: Set[Feature[_]] = Set.empty override def setFeatures( contentFeatures: Option[ContentFeatures], richDataRecord: RichDataRecord ): Unit = { if (contentFeatures.nonEmpty) { val features = contentFeatures.get richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.ASPECT_RATIO_DEN, features.aspectRatioNum.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.ASPECT_RATIO_NUM, features.aspectRatioNum.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.HEIGHT_1, features.heights.flatMap(_.lift(0)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.HEIGHT_2, features.heights.flatMap(_.lift(1)).map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.VIDEO_DURATION, features.videoDurationMs.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.NUM_CAPS, Some(features.numCaps.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.TWEET_LENGTH, Some(features.length.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.HAS_QUESTION, Some(features.hasQuestion) ) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/scala/com/twitter/timelines/prediction/features/common", "src/scala/com/twitter/timelines/prediction/features/recap", "src/scala/com/twitter/timelines/util", "src/thrift/com/twitter/search/common:features-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.ml.api.util.DataRecordConverters._ import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.search.common.features.{thriftscala => sc} import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import com.twitter.timelines.prediction.features.recap.RecapFeatures import com.twitter.timelines.util.UrlExtractorUtil import java.lang.{Boolean => JBoolean} import java.lang.{Double => JDouble} import java.util.{Map => JMap} import scala.collection.JavaConverters._ object EarlybirdAdapter extends TimelinesMutatingAdapterBase[Option[sc.ThriftTweetFeatures]] { override val getFeatureContext: FeatureContext = new FeatureContext( RecapFeatures.BIDIRECTIONAL_FAV_COUNT, RecapFeatures.BIDIRECTIONAL_REPLY_COUNT, RecapFeatures.BIDIRECTIONAL_RETWEET_COUNT, RecapFeatures.BLENDER_SCORE, RecapFeatures.CONTAINS_MEDIA, RecapFeatures.CONVERSATIONAL_COUNT, RecapFeatures.EMBEDS_IMPRESSION_COUNT, RecapFeatures.EMBEDS_URL_COUNT, RecapFeatures.FAV_COUNT, RecapFeatures.FAV_COUNT_V2, RecapFeatures.FROM_INACTIVE_USER, RecapFeatures.FROM_MUTUAL_FOLLOW, RecapFeatures.FROM_VERIFIED_ACCOUNT, RecapFeatures.HAS_CARD, RecapFeatures.HAS_CONSUMER_VIDEO, RecapFeatures.HAS_HASHTAG, // RecapFeatures.HAS_IMAGE, RecapFeatures.HAS_LINK, RecapFeatures.HAS_MENTION, RecapFeatures.HAS_MULTIPLE_HASHTAGS_OR_TRENDS, RecapFeatures.HAS_MULTIPLE_MEDIA, RecapFeatures.HAS_NATIVE_IMAGE, RecapFeatures.HAS_NATIVE_VIDEO, RecapFeatures.HAS_NEWS, RecapFeatures.HAS_PERISCOPE, RecapFeatures.HAS_PRO_VIDEO, RecapFeatures.HAS_TREND, // RecapFeatures.HAS_VIDEO, RecapFeatures.HAS_VINE, RecapFeatures.HAS_VISIBLE_LINK, RecapFeatures.IS_AUTHOR_BOT, RecapFeatures.IS_AUTHOR_NEW, RecapFeatures.IS_AUTHOR_NSFW, RecapFeatures.IS_AUTHOR_PROFILE_EGG, RecapFeatures.IS_AUTHOR_SPAM, RecapFeatures.IS_BUSINESS_SCORE, RecapFeatures.IS_OFFENSIVE, RecapFeatures.IS_REPLY, RecapFeatures.IS_RETWEET, RecapFeatures.IS_RETWEETER_BOT, RecapFeatures.IS_RETWEETER_NEW, RecapFeatures.IS_RETWEETER_NSFW, RecapFeatures.IS_RETWEETER_PROFILE_EGG, RecapFeatures.IS_RETWEETER_SPAM, RecapFeatures.IS_RETWEET_OF_REPLY, RecapFeatures.IS_SENSITIVE, RecapFeatures.LANGUAGE, RecapFeatures.LINK_COUNT, RecapFeatures.LINK_LANGUAGE, RecapFeatures.MATCH_SEARCHER_LANGS, RecapFeatures.MATCH_SEARCHER_MAIN_LANG, RecapFeatures.MATCH_UI_LANG, RecapFeatures.MENTIONED_SCREEN_NAMES, RecapFeatures.MENTION_SEARCHER, RecapFeatures.NUM_HASHTAGS, RecapFeatures.NUM_MENTIONS, RecapFeatures.PREV_USER_TWEET_ENGAGEMENT, RecapFeatures.PROBABLY_FROM_FOLLOWED_AUTHOR, RecapFeatures.REPLY_COUNT, RecapFeatures.REPLY_COUNT_V2, RecapFeatures.REPLY_OTHER, RecapFeatures.REPLY_SEARCHER, RecapFeatures.RETWEET_COUNT, RecapFeatures.RETWEET_COUNT_V2, RecapFeatures.RETWEET_DIRECTED_AT_USER_IN_FIRST_DEGREE, RecapFeatures.RETWEET_OF_MUTUAL_FOLLOW, RecapFeatures.RETWEET_OTHER, RecapFeatures.RETWEET_SEARCHER, RecapFeatures.SIGNATURE, RecapFeatures.SOURCE_AUTHOR_REP, RecapFeatures.TEXT_SCORE, RecapFeatures.TWEET_COUNT_FROM_USER_IN_SNAPSHOT, RecapFeatures.UNIDIRECTIONAL_FAV_COUNT, RecapFeatures.UNIDIRECTIONAL_REPLY_COUNT, RecapFeatures.UNIDIRECTIONAL_RETWEET_COUNT, RecapFeatures.URL_DOMAINS, RecapFeatures.USER_REP, RecapFeatures.VIDEO_VIEW_COUNT, // shared features TimelinesSharedFeatures.WEIGHTED_FAV_COUNT, TimelinesSharedFeatures.WEIGHTED_RETWEET_COUNT, TimelinesSharedFeatures.WEIGHTED_REPLY_COUNT, TimelinesSharedFeatures.WEIGHTED_QUOTE_COUNT, TimelinesSharedFeatures.EMBEDS_IMPRESSION_COUNT_V2, TimelinesSharedFeatures.EMBEDS_URL_COUNT_V2, TimelinesSharedFeatures.DECAYED_FAVORITE_COUNT, TimelinesSharedFeatures.DECAYED_RETWEET_COUNT, TimelinesSharedFeatures.DECAYED_REPLY_COUNT, TimelinesSharedFeatures.DECAYED_QUOTE_COUNT, TimelinesSharedFeatures.FAKE_FAVORITE_COUNT, TimelinesSharedFeatures.FAKE_RETWEET_COUNT, TimelinesSharedFeatures.FAKE_REPLY_COUNT, TimelinesSharedFeatures.FAKE_QUOTE_COUNT, TimelinesSharedFeatures.QUOTE_COUNT, TimelinesSharedFeatures.EARLYBIRD_SCORE, // Safety features TimelinesSharedFeatures.LABEL_ABUSIVE_FLAG, TimelinesSharedFeatures.LABEL_ABUSIVE_HI_RCL_FLAG, TimelinesSharedFeatures.LABEL_DUP_CONTENT_FLAG, TimelinesSharedFeatures.LABEL_NSFW_HI_PRC_FLAG, TimelinesSharedFeatures.LABEL_NSFW_HI_RCL_FLAG, TimelinesSharedFeatures.LABEL_SPAM_FLAG, TimelinesSharedFeatures.LABEL_SPAM_HI_RCL_FLAG, // periscope features TimelinesSharedFeatures.PERISCOPE_EXISTS, TimelinesSharedFeatures.PERISCOPE_IS_LIVE, TimelinesSharedFeatures.PERISCOPE_HAS_BEEN_FEATURED, TimelinesSharedFeatures.PERISCOPE_IS_CURRENTLY_FEATURED, TimelinesSharedFeatures.PERISCOPE_IS_FROM_QUALITY_SOURCE, // VISIBLE_TOKEN_RATIO TimelinesSharedFeatures.VISIBLE_TOKEN_RATIO, TimelinesSharedFeatures.HAS_QUOTE, TimelinesSharedFeatures.IS_COMPOSER_SOURCE_CAMERA, // health features TimelinesSharedFeatures.PREPORTED_TWEET_SCORE, // media TimelinesSharedFeatures.CLASSIFICATION_LABELS ) override val commonFeatures: Set[Feature[_]] = Set.empty override def setFeatures( ebFeatures: Option[sc.ThriftTweetFeatures], richDataRecord: RichDataRecord ): Unit = { if (ebFeatures.nonEmpty) { val features = ebFeatures.get richDataRecord.setFeatureValue[JDouble]( RecapFeatures.PREV_USER_TWEET_ENGAGEMENT, features.prevUserTweetEngagement.toDouble ) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.IS_SENSITIVE, features.isSensitiveContent) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.HAS_MULTIPLE_MEDIA, features.hasMultipleMedia) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.IS_AUTHOR_PROFILE_EGG, features.isAuthorProfileEgg) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_AUTHOR_NEW, features.isAuthorNew) richDataRecord .setFeatureValue[JDouble](RecapFeatures.NUM_MENTIONS, features.numMentions.toDouble) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_MENTION, features.numMentions > 0) richDataRecord .setFeatureValue[JDouble](RecapFeatures.NUM_HASHTAGS, features.numHashtags.toDouble) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_HASHTAG, features.numHashtags > 0) richDataRecord .setFeatureValue[JDouble](RecapFeatures.LINK_LANGUAGE, features.linkLanguage.toDouble) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_AUTHOR_NSFW, features.isAuthorNSFW) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_AUTHOR_SPAM, features.isAuthorSpam) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_AUTHOR_BOT, features.isAuthorBot) richDataRecord.setFeatureValueFromOption( RecapFeatures.LANGUAGE, features.language.map(_.getValue.toLong)) richDataRecord.setFeatureValueFromOption( RecapFeatures.SIGNATURE, features.signature.map(_.toLong)) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.FROM_INACTIVE_USER, features.fromInActiveUser) richDataRecord .setFeatureValue[JBoolean]( RecapFeatures.PROBABLY_FROM_FOLLOWED_AUTHOR, features.probablyFromFollowedAuthor) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.FROM_MUTUAL_FOLLOW, features.fromMutualFollow) richDataRecord.setFeatureValue[JBoolean]( RecapFeatures.FROM_VERIFIED_ACCOUNT, features.fromVerifiedAccount) richDataRecord.setFeatureValue[JDouble](RecapFeatures.USER_REP, features.userRep) richDataRecord .setFeatureValue[JDouble](RecapFeatures.IS_BUSINESS_SCORE, features.isBusinessScore) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.HAS_CONSUMER_VIDEO, features.hasConsumerVideo) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_PRO_VIDEO, features.hasProVideo) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_VINE, features.hasVine) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_PERISCOPE, features.hasPeriscope) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.HAS_NATIVE_VIDEO, features.hasNativeVideo) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.HAS_NATIVE_IMAGE, features.hasNativeImage) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_CARD, features.hasCard) // richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_IMAGE, features.hasImage) //handled via TweetEntityServiceContentFeatureHydrator richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_NEWS, features.hasNews) // richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_VIDEO, features.hasVideo) //handled via TweetEntityServiceContentFeatureHydrator richDataRecord.setFeatureValue[JBoolean](RecapFeatures.CONTAINS_MEDIA, features.containsMedia) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.RETWEET_SEARCHER, features.retweetSearcher) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.REPLY_SEARCHER, features.replySearcher) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.MENTION_SEARCHER, features.mentionSearcher) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.REPLY_OTHER, features.replyOther) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.RETWEET_OTHER, features.retweetOther) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_REPLY, features.isReply) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_RETWEET, features.isRetweet) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_OFFENSIVE, features.isOffensive) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.MATCH_UI_LANG, features.matchesUILang) richDataRecord .setFeatureValue[JBoolean]( RecapFeatures.MATCH_SEARCHER_MAIN_LANG, features.matchesSearcherMainLang) richDataRecord.setFeatureValue[JBoolean]( RecapFeatures.MATCH_SEARCHER_LANGS, features.matchesSearcherLangs) richDataRecord .setFeatureValue[JDouble]( RecapFeatures.BIDIRECTIONAL_FAV_COUNT, features.bidirectionalFavCount) richDataRecord .setFeatureValue[JDouble]( RecapFeatures.UNIDIRECTIONAL_FAV_COUNT, features.unidirectionalFavCount) richDataRecord .setFeatureValue[JDouble]( RecapFeatures.BIDIRECTIONAL_REPLY_COUNT, features.bidirectionalReplyCount) richDataRecord .setFeatureValue[JDouble]( RecapFeatures.UNIDIRECTIONAL_REPLY_COUNT, features.unidirectionalReplyCount) richDataRecord .setFeatureValue[JDouble]( RecapFeatures.BIDIRECTIONAL_RETWEET_COUNT, features.bidirectionalRetweetCount) richDataRecord .setFeatureValue[JDouble]( RecapFeatures.UNIDIRECTIONAL_RETWEET_COUNT, features.unidirectionalRetweetCount) richDataRecord .setFeatureValue[JDouble](RecapFeatures.CONVERSATIONAL_COUNT, features.conversationCount) richDataRecord.setFeatureValue[JDouble]( RecapFeatures.TWEET_COUNT_FROM_USER_IN_SNAPSHOT, features.tweetCountFromUserInSnapshot ) richDataRecord .setFeatureValue[JBoolean]( RecapFeatures.IS_RETWEETER_PROFILE_EGG, features.isRetweeterProfileEgg) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.IS_RETWEETER_NEW, features.isRetweeterNew) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.IS_RETWEETER_BOT, features.isRetweeterBot) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.IS_RETWEETER_NSFW, features.isRetweeterNSFW) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.IS_RETWEETER_SPAM, features.isRetweeterSpam) richDataRecord .setFeatureValue[JBoolean]( RecapFeatures.RETWEET_OF_MUTUAL_FOLLOW, features.retweetOfMutualFollow) richDataRecord .setFeatureValue[JDouble](RecapFeatures.SOURCE_AUTHOR_REP, features.sourceAuthorRep) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.IS_RETWEET_OF_REPLY, features.isRetweetOfReply) richDataRecord.setFeatureValueFromOption( RecapFeatures.RETWEET_DIRECTED_AT_USER_IN_FIRST_DEGREE, features.retweetDirectedAtUserInFirstDegree ) richDataRecord .setFeatureValue[JDouble]( RecapFeatures.EMBEDS_IMPRESSION_COUNT, features.embedsImpressionCount.toDouble) richDataRecord .setFeatureValue[JDouble](RecapFeatures.EMBEDS_URL_COUNT, features.embedsUrlCount.toDouble) richDataRecord .setFeatureValue[JDouble](RecapFeatures.VIDEO_VIEW_COUNT, features.videoViewCount.toDouble) richDataRecord .setFeatureValue[JDouble](RecapFeatures.REPLY_COUNT, features.replyCount.toDouble) richDataRecord .setFeatureValue[JDouble](RecapFeatures.RETWEET_COUNT, features.retweetCount.toDouble) richDataRecord.setFeatureValue[JDouble](RecapFeatures.FAV_COUNT, features.favCount.toDouble) richDataRecord.setFeatureValue[JDouble](RecapFeatures.BLENDER_SCORE, features.blenderScore) richDataRecord.setFeatureValue[JDouble](RecapFeatures.TEXT_SCORE, features.textScore) richDataRecord .setFeatureValue[JBoolean](RecapFeatures.HAS_VISIBLE_LINK, features.hasVisibleLink) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_LINK, features.hasLink) richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_TREND, features.hasTrend) richDataRecord.setFeatureValue[JBoolean]( RecapFeatures.HAS_MULTIPLE_HASHTAGS_OR_TRENDS, features.hasMultipleHashtagsOrTrends ) richDataRecord.setFeatureValueFromOption( RecapFeatures.FAV_COUNT_V2, features.favCountV2.map(_.toDouble)) richDataRecord.setFeatureValueFromOption( RecapFeatures.RETWEET_COUNT_V2, features.retweetCountV2.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( RecapFeatures.REPLY_COUNT_V2, features.replyCountV2.map(_.toDouble)) richDataRecord.setFeatureValueFromOption( RecapFeatures.MENTIONED_SCREEN_NAMES, features.mentionsList.map(_.toSet.asJava) ) val urls = features.urlsList.getOrElse(Seq.empty) richDataRecord.setFeatureValue( RecapFeatures.URL_DOMAINS, urls.toSet.flatMap(UrlExtractorUtil.extractDomain).asJava) richDataRecord.setFeatureValue[JDouble](RecapFeatures.LINK_COUNT, urls.size.toDouble) // shared features richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.WEIGHTED_FAV_COUNT, features.weightedFavoriteCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.WEIGHTED_RETWEET_COUNT, features.weightedRetweetCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.WEIGHTED_REPLY_COUNT, features.weightedReplyCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.WEIGHTED_QUOTE_COUNT, features.weightedQuoteCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.EMBEDS_IMPRESSION_COUNT_V2, features.embedsImpressionCountV2.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.EMBEDS_URL_COUNT_V2, features.embedsUrlCountV2.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.DECAYED_FAVORITE_COUNT, features.decayedFavoriteCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.DECAYED_RETWEET_COUNT, features.decayedRetweetCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.DECAYED_REPLY_COUNT, features.decayedReplyCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.DECAYED_QUOTE_COUNT, features.decayedQuoteCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.FAKE_FAVORITE_COUNT, features.fakeFavoriteCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.FAKE_RETWEET_COUNT, features.fakeRetweetCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.FAKE_REPLY_COUNT, features.fakeReplyCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.FAKE_QUOTE_COUNT, features.fakeQuoteCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.QUOTE_COUNT, features.quoteCount.map(_.toDouble) ) richDataRecord.setFeatureValue[JDouble]( TimelinesSharedFeatures.EARLYBIRD_SCORE, features.earlybirdScore ) // safety features richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.LABEL_ABUSIVE_FLAG, features.labelAbusiveFlag ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.LABEL_ABUSIVE_HI_RCL_FLAG, features.labelAbusiveHiRclFlag ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.LABEL_DUP_CONTENT_FLAG, features.labelDupContentFlag ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.LABEL_NSFW_HI_PRC_FLAG, features.labelNsfwHiPrcFlag ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.LABEL_NSFW_HI_RCL_FLAG, features.labelNsfwHiRclFlag ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.LABEL_SPAM_FLAG, features.labelSpamFlag ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.LABEL_SPAM_HI_RCL_FLAG, features.labelSpamHiRclFlag ) // periscope features richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.PERISCOPE_EXISTS, features.periscopeExists ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.PERISCOPE_IS_LIVE, features.periscopeIsLive ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.PERISCOPE_HAS_BEEN_FEATURED, features.periscopeHasBeenFeatured ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.PERISCOPE_IS_CURRENTLY_FEATURED, features.periscopeIsCurrentlyFeatured ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.PERISCOPE_IS_FROM_QUALITY_SOURCE, features.periscopeIsFromQualitySource ) // misc features richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.VISIBLE_TOKEN_RATIO, features.visibleTokenRatio.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.HAS_QUOTE, features.hasQuote ) richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.IS_COMPOSER_SOURCE_CAMERA, features.isComposerSourceCamera ) // health scores richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.PREPORTED_TWEET_SCORE, features.pReportedTweetScore ) // media richDataRecord.setFeatureValueFromOption( TimelinesSharedFeatures.CLASSIFICATION_LABELS, features.mediaClassificationInfo.map(_.toMap.asJava.asInstanceOf[JMap[String, JDouble]]) ) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/InReplyToEarlybirdAdapter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.ml.api.util.DataRecordConverters.RichDataRecordWrapper import com.twitter.search.common.features.{thriftscala => sc} import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.features.common.InReplyToTweetTimelinesSharedFeatures import com.twitter.timelines.prediction.features.recap.InReplyToRecapFeatures import java.lang.{Boolean => JBoolean} import java.lang.{Double => JDouble} object InReplyToEarlybirdAdapter extends TimelinesMutatingAdapterBase[Option[sc.ThriftTweetFeatures]] { override val getFeatureContext: FeatureContext = new FeatureContext( // TextFeatures InReplyToTweetTimelinesSharedFeatures.WEIGHTED_FAV_COUNT, InReplyToTweetTimelinesSharedFeatures.WEIGHTED_RETWEET_COUNT, InReplyToTweetTimelinesSharedFeatures.WEIGHTED_REPLY_COUNT, InReplyToTweetTimelinesSharedFeatures.WEIGHTED_QUOTE_COUNT, InReplyToTweetTimelinesSharedFeatures.DECAYED_FAVORITE_COUNT, InReplyToTweetTimelinesSharedFeatures.DECAYED_RETWEET_COUNT, InReplyToTweetTimelinesSharedFeatures.DECAYED_REPLY_COUNT, InReplyToTweetTimelinesSharedFeatures.DECAYED_QUOTE_COUNT, InReplyToTweetTimelinesSharedFeatures.QUOTE_COUNT, InReplyToTweetTimelinesSharedFeatures.HAS_QUOTE, InReplyToTweetTimelinesSharedFeatures.EARLYBIRD_SCORE, InReplyToRecapFeatures.PREV_USER_TWEET_ENGAGEMENT, InReplyToRecapFeatures.IS_SENSITIVE, InReplyToRecapFeatures.IS_AUTHOR_NEW, InReplyToRecapFeatures.NUM_MENTIONS, InReplyToRecapFeatures.HAS_MENTION, InReplyToRecapFeatures.HAS_HASHTAG, InReplyToRecapFeatures.IS_AUTHOR_NSFW, InReplyToRecapFeatures.IS_AUTHOR_SPAM, InReplyToRecapFeatures.IS_AUTHOR_BOT, InReplyToRecapFeatures.FROM_MUTUAL_FOLLOW, InReplyToRecapFeatures.USER_REP, InReplyToRecapFeatures.FROM_VERIFIED_ACCOUNT, InReplyToRecapFeatures.HAS_IMAGE, InReplyToRecapFeatures.HAS_NEWS, InReplyToRecapFeatures.HAS_VIDEO, InReplyToRecapFeatures.HAS_VISIBLE_LINK, InReplyToRecapFeatures.IS_OFFENSIVE, InReplyToRecapFeatures.IS_REPLY, InReplyToRecapFeatures.BIDIRECTIONAL_REPLY_COUNT, InReplyToRecapFeatures.UNIDIRECTIONAL_REPLY_COUNT, InReplyToRecapFeatures.BIDIRECTIONAL_RETWEET_COUNT, InReplyToRecapFeatures.UNIDIRECTIONAL_RETWEET_COUNT, InReplyToRecapFeatures.BIDIRECTIONAL_FAV_COUNT, InReplyToRecapFeatures.UNIDIRECTIONAL_FAV_COUNT, InReplyToRecapFeatures.CONVERSATIONAL_COUNT, InReplyToRecapFeatures.REPLY_COUNT, InReplyToRecapFeatures.RETWEET_COUNT, InReplyToRecapFeatures.FAV_COUNT, InReplyToRecapFeatures.TEXT_SCORE, InReplyToRecapFeatures.FAV_COUNT_V2, InReplyToRecapFeatures.RETWEET_COUNT_V2, InReplyToRecapFeatures.REPLY_COUNT_V2) override val commonFeatures: Set[Feature[_]] = Set.empty override def setFeatures( ebFeatures: Option[sc.ThriftTweetFeatures], richDataRecord: RichDataRecord ): Unit = { if (ebFeatures.nonEmpty) { val features = ebFeatures.get richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.WEIGHTED_FAV_COUNT, features.weightedFavoriteCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.WEIGHTED_RETWEET_COUNT, features.weightedRetweetCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.WEIGHTED_REPLY_COUNT, features.weightedReplyCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.WEIGHTED_QUOTE_COUNT, features.weightedQuoteCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.DECAYED_FAVORITE_COUNT, features.decayedFavoriteCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.DECAYED_RETWEET_COUNT, features.decayedRetweetCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.DECAYED_REPLY_COUNT, features.decayedReplyCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.DECAYED_QUOTE_COUNT, features.decayedQuoteCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.QUOTE_COUNT, features.quoteCount.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToTweetTimelinesSharedFeatures.HAS_QUOTE, features.hasQuote ) if (features.earlybirdScore > 0) richDataRecord.setFeatureValue[JDouble]( InReplyToTweetTimelinesSharedFeatures.EARLYBIRD_SCORE, features.earlybirdScore ) richDataRecord.setFeatureValue[JDouble]( InReplyToRecapFeatures.PREV_USER_TWEET_ENGAGEMENT, features.prevUserTweetEngagement.toDouble ) richDataRecord .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_SENSITIVE, features.isSensitiveContent) richDataRecord .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_AUTHOR_NEW, features.isAuthorNew) richDataRecord.setFeatureValue[JDouble]( InReplyToRecapFeatures.NUM_MENTIONS, features.numMentions.toDouble) richDataRecord .setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_MENTION, (features.numMentions > 0)) richDataRecord .setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_HASHTAG, (features.numHashtags > 0)) richDataRecord .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_AUTHOR_NSFW, features.isAuthorNSFW) richDataRecord .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_AUTHOR_SPAM, features.isAuthorSpam) richDataRecord .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_AUTHOR_BOT, features.isAuthorBot) richDataRecord.setFeatureValue[JBoolean]( InReplyToRecapFeatures.FROM_MUTUAL_FOLLOW, features.fromMutualFollow) richDataRecord.setFeatureValue[JDouble](InReplyToRecapFeatures.USER_REP, features.userRep) richDataRecord.setFeatureValue[JBoolean]( InReplyToRecapFeatures.FROM_VERIFIED_ACCOUNT, features.fromVerifiedAccount) richDataRecord.setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_IMAGE, features.hasImage) richDataRecord.setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_NEWS, features.hasNews) richDataRecord.setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_VIDEO, features.hasVideo) richDataRecord .setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_VISIBLE_LINK, features.hasVisibleLink) richDataRecord .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_OFFENSIVE, features.isOffensive) richDataRecord.setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_REPLY, features.isReply) richDataRecord.setFeatureValue[JDouble]( InReplyToRecapFeatures.BIDIRECTIONAL_REPLY_COUNT, features.bidirectionalReplyCount) richDataRecord.setFeatureValue[JDouble]( InReplyToRecapFeatures.UNIDIRECTIONAL_REPLY_COUNT, features.unidirectionalReplyCount) richDataRecord.setFeatureValue[JDouble]( InReplyToRecapFeatures.BIDIRECTIONAL_RETWEET_COUNT, features.bidirectionalRetweetCount) richDataRecord.setFeatureValue[JDouble]( InReplyToRecapFeatures.UNIDIRECTIONAL_RETWEET_COUNT, features.unidirectionalRetweetCount) richDataRecord.setFeatureValue[JDouble]( InReplyToRecapFeatures.BIDIRECTIONAL_FAV_COUNT, features.bidirectionalFavCount) richDataRecord.setFeatureValue[JDouble]( InReplyToRecapFeatures.UNIDIRECTIONAL_FAV_COUNT, features.unidirectionalFavCount) richDataRecord.setFeatureValue[JDouble]( InReplyToRecapFeatures.CONVERSATIONAL_COUNT, features.conversationCount) richDataRecord .setFeatureValue[JDouble](InReplyToRecapFeatures.REPLY_COUNT, features.replyCount.toDouble) richDataRecord.setFeatureValue[JDouble]( InReplyToRecapFeatures.RETWEET_COUNT, features.retweetCount.toDouble) richDataRecord .setFeatureValue[JDouble](InReplyToRecapFeatures.FAV_COUNT, features.favCount.toDouble) richDataRecord.setFeatureValue[JDouble](InReplyToRecapFeatures.TEXT_SCORE, features.textScore) richDataRecord.setFeatureValueFromOption( InReplyToRecapFeatures.FAV_COUNT_V2, features.favCountV2.map(_.toDouble)) richDataRecord.setFeatureValueFromOption( InReplyToRecapFeatures.RETWEET_COUNT_V2, features.retweetCountV2.map(_.toDouble) ) richDataRecord.setFeatureValueFromOption( InReplyToRecapFeatures.REPLY_COUNT_V2, features.replyCountV2.map(_.toDouble)) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/scala/com/twitter/timelines/prediction/features/common", "src/thrift/com/twitter/ml/api:data-java", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.inferred_topic import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import scala.collection.JavaConverters._ object InferredTopicAdapter extends TimelinesMutatingAdapterBase[Map[Long, Double]] { override val getFeatureContext: FeatureContext = new FeatureContext( TimelinesSharedFeatures.INFERRED_TOPIC_IDS) override val commonFeatures: Set[Feature[_]] = Set.empty override def setFeatures( inferredTopicFeatures: Map[Long, Double], richDataRecord: RichDataRecord ): Unit = { richDataRecord.setFeatureValue( TimelinesSharedFeatures.INFERRED_TOPIC_IDS, inferredTopicFeatures.keys.map(_.toString).toSet.asJava) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "src/java/com/twitter/ml/api:api-base", "src/java/com/twitter/ml/api/constant", "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/scala/com/twitter/timelines/prediction/features/common", "src/thrift/com/twitter/ml/api:data-java", "src/thrift/com/twitter/timelines/author_features:thrift-java", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features import com.twitter.ml.api.constant.SharedFeatures import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import java.lang.{Long => JLong} case class NonMLCandidateFeatures( tweetId: Long, sourceTweetId: Option[Long], originalAuthorId: Option[Long], ) /** * define non ml features adapter to create a data record which includes many non ml features * e.g. predictionRequestId, userId, tweetId to be used as joined key in batch pipeline. */ object NonMLCandidateFeaturesAdapter extends TimelinesMutatingAdapterBase[NonMLCandidateFeatures] { private val featureContext = new FeatureContext( SharedFeatures.TWEET_ID, // For Secondary Engagement data generation TimelinesSharedFeatures.SOURCE_TWEET_ID, TimelinesSharedFeatures.ORIGINAL_AUTHOR_ID, ) override def getFeatureContext: FeatureContext = featureContext override val commonFeatures: Set[Feature[_]] = Set.empty override def setFeatures( nonMLCandidateFeatures: NonMLCandidateFeatures, richDataRecord: RichDataRecord ): Unit = { richDataRecord.setFeatureValue[JLong](SharedFeatures.TWEET_ID, nonMLCandidateFeatures.tweetId) nonMLCandidateFeatures.sourceTweetId.foreach( richDataRecord.setFeatureValue[JLong](TimelinesSharedFeatures.SOURCE_TWEET_ID, _)) nonMLCandidateFeatures.originalAuthorId.foreach( richDataRecord.setFeatureValue[JLong](TimelinesSharedFeatures.ORIGINAL_AUTHOR_ID, _)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features import com.twitter.ml.api.constant.SharedFeatures import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import java.lang.{Long => JLong} case class NonMLCommonFeatures( userId: Long, predictionRequestId: Option[Long], servedTimestamp: Long, ) /** * define non ml features adapter to create a data record which includes many non ml features * e.g. predictionRequestId, userId, tweetId to be used as joined key in batch pipeline. */ object NonMLCommonFeaturesAdapter extends TimelinesMutatingAdapterBase[NonMLCommonFeatures] { private val featureContext = new FeatureContext( SharedFeatures.USER_ID, TimelinesSharedFeatures.PREDICTION_REQUEST_ID, TimelinesSharedFeatures.SERVED_TIMESTAMP, ) override def getFeatureContext: FeatureContext = featureContext override val commonFeatures: Set[Feature[_]] = Set( SharedFeatures.USER_ID, TimelinesSharedFeatures.PREDICTION_REQUEST_ID, TimelinesSharedFeatures.SERVED_TIMESTAMP, ) override def setFeatures( nonMLCommonFeatures: NonMLCommonFeatures, richDataRecord: RichDataRecord ): Unit = { richDataRecord.setFeatureValue[JLong](SharedFeatures.USER_ID, nonMLCommonFeatures.userId) nonMLCommonFeatures.predictionRequestId.foreach( richDataRecord.setFeatureValue[JLong](TimelinesSharedFeatures.PREDICTION_REQUEST_ID, _)) richDataRecord.setFeatureValue[JLong]( TimelinesSharedFeatures.SERVED_TIMESTAMP, nonMLCommonFeatures.servedTimestamp) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/thrift/com/twitter/ml/api:data-java", "timelines/data_processing/ml_util/aggregation_framework/conversion:for-timelines", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.offline_aggregates import com.twitter.ml.api.DataRecord import com.twitter.ml.api.IRecordOneToOneAdapter object PassThroughAdapter extends IRecordOneToOneAdapter[Seq[DataRecord]] { override def adaptToDataRecord(record: Seq[DataRecord]): DataRecord = record.headOption.getOrElse(new DataRecord) // This is not necessary and should not be used. override def getFeatureContext = ??? } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.offline_aggregates import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.timelines.data_processing.ml_util.aggregation_framework.conversion.CombineCountsPolicy import com.twitter.timelines.prediction.common.adapters.TimelinesIRecordAdapter class SparseAggregatesToDenseAdapter(policy: CombineCountsPolicy) extends TimelinesIRecordAdapter[Seq[DataRecord]] { override def setFeatures(input: Seq[DataRecord], mutableDataRecord: RichDataRecord): Unit = policy.defaultMergeRecord(mutableDataRecord.getRecord, input.toList) override val getFeatureContext: FeatureContext = new FeatureContext(policy.outputFeaturesPostMerge.toSeq: _*) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/ml/api:api-base", "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/thrift/com/twitter/ml/api:data-java", "src/thrift/com/twitter/ml/api:data-scala", "src/thrift/com/twitter/ml/api:embedding-java", "src/thrift/com/twitter/ml/api:embedding-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings import com.twitter.ml.api.DataType import com.twitter.ml.api.Feature import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions import com.twitter.ml.api.{thriftscala => ml} import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase sealed trait TwhinEmbeddingsAdapter extends TimelinesMutatingAdapterBase[Option[ml.FloatTensor]] { def twhinEmbeddingsFeature: Feature.Tensor override def getFeatureContext: FeatureContext = new FeatureContext( twhinEmbeddingsFeature ) override def setFeatures( embedding: Option[ml.FloatTensor], richDataRecord: RichDataRecord ): Unit = { embedding.foreach { floatTensor => richDataRecord.setFeatureValue( twhinEmbeddingsFeature, ScalaToJavaDataRecordConversions.scalaTensor2Java( ml.GeneralTensor .FloatTensor(floatTensor))) } } } object TwhinEmbeddingsFeatures { val TwhinAuthorFollowEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( "original_author.twhin.tw_hi_n.author_follow_as_float_tensor", DataType.FLOAT ) val TwhinUserEngagementEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( "user.twhin.tw_hi_n.user_engagement_as_float_tensor", DataType.FLOAT ) val TwhinUserFollowEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( "user.twhin.tw_hi_n.user_follow_as_float_tensor", DataType.FLOAT ) } object TwhinAuthorFollowEmbeddingsAdapter extends TwhinEmbeddingsAdapter { override val twhinEmbeddingsFeature: Feature.Tensor = TwhinEmbeddingsFeatures.TwhinAuthorFollowEmbeddingsFeature override val commonFeatures: Set[Feature[_]] = Set.empty } object TwhinUserEngagementEmbeddingsAdapter extends TwhinEmbeddingsAdapter { override val twhinEmbeddingsFeature: Feature.Tensor = TwhinEmbeddingsFeatures.TwhinUserEngagementEmbeddingsFeature override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) } object TwhinUserFollowEmbeddingsAdapter extends TwhinEmbeddingsAdapter { override val twhinEmbeddingsFeature: Feature.Tensor = TwhinEmbeddingsFeatures.TwhinUserFollowEmbeddingsFeature override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.ml.api.FeatureContext import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType.AggregateType import com.twitter.timelines.data_processing.ml_util.aggregation_framework.TypedAggregateGroup import scala.jdk.CollectionConverters.asJavaIterableConverter // A helper class deriving aggregate feature info from the given configuration parameters. class AggregateFeatureInfo( val aggregateGroups: Set[AggregateGroup], val aggregateType: AggregateType) { private val typedAggregateGroups = aggregateGroups.flatMap(_.buildTypedAggregateGroups()).toList val featureContext: FeatureContext = new FeatureContext( (typedAggregateGroups.flatMap(_.allOutputFeatures) ++ typedAggregateGroups.flatMap(_.allOutputKeys) ++ Seq(TypedAggregateGroup.timestampFeature)).asJava) val feature: BaseAggregateRootFeature = AggregateFeatureInfo.pickFeature(aggregateType) } object AggregateFeatureInfo { val features: Set[BaseAggregateRootFeature] = Set(PartAAggregateRootFeature, PartBAggregateRootFeature) def pickFeature(aggregateType: AggregateType): BaseAggregateRootFeature = { val filtered = features.filter(_.aggregateTypes.contains(aggregateType)) require( filtered.size == 1, "requested AggregateType must be backed by exactly one physical store.") filtered.head } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeaturesToDecodeWithMetadata.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.timelinemixer.injection.repository.uss.VersionedAggregateFeaturesDecoder import com.twitter.ml.api.DataRecord import com.twitter.timelines.aggregate_interactions.thriftjava.UserAggregateInteractions import com.twitter.timelines.aggregate_interactions.v17.thriftjava.{ UserAggregateInteractions => V17UserAggregateInteractions } import com.twitter.timelines.aggregate_interactions.v1.thriftjava.{ UserAggregateInteractions => V1UserAggregateInteractions } import com.twitter.timelines.suggests.common.dense_data_record.thriftjava.DenseCompactDataRecord import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata import java.lang.{Long => JLong} import java.util.Collections import java.util.{Map => JMap} private[offline_aggregates] case class AggregateFeaturesToDecodeWithMetadata( metadataOpt: Option[DenseFeatureMetadata], aggregates: UserAggregateInteractions) { def toDataRecord(dr: DenseCompactDataRecord): DataRecord = VersionedAggregateFeaturesDecoder.fromJDenseCompact( metadataOpt, dr.versionId, NullStatsReceiver, s"V${dr.versionId}" )(dr) def userAggregatesOpt: Option[DenseCompactDataRecord] = { aggregates.getSetField match { case UserAggregateInteractions._Fields.V17 => Option(aggregates.getV17.user_aggregates) case _ => None } } def userAuthorAggregates = extract(_.user_author_aggregates) def userEngagerAggregates = extract(_.user_engager_aggregates) def userMentionAggregates = extract(_.user_mention_aggregates) def userOriginalAuthorAggregates = extract(_.user_original_author_aggregates) def userRequestDowAggregates = extract(_.user_request_dow_aggregates) def userRequestHourAggregates = extract(_.user_request_hour_aggregates) def rectweetUserSimclustersTweetAggregates = extract(_.rectweet_user_simclusters_tweet_aggregates) def userTwitterListAggregates = extract(_.user_list_aggregates) def userTopicAggregates = extract(_.user_topic_aggregates) def userInferredTopicAggregates = extract(_.user_inferred_topic_aggregates) def userMediaUnderstandingAnnotationAggregates = extract( _.user_media_understanding_annotation_aggregates) private def extract[T]( v17Fn: V17UserAggregateInteractions => JMap[JLong, DenseCompactDataRecord] ): JMap[JLong, DenseCompactDataRecord] = { aggregates.getSetField match { case UserAggregateInteractions._Fields.V17 => v17Fn(aggregates.getV17) case _ => Collections.emptyMap() } } } object AggregateFeaturesToDecodeWithMetadata { val empty = new AggregateFeaturesToDecodeWithMetadata( None, UserAggregateInteractions.v1(new V1UserAggregateInteractions())) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "finatra/inject/inject-core/src/main/scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "servo/repo/src/main/scala", "src/java/com/twitter/ml/api:api-base", "src/scala/com/twitter/timelines/prediction/adapters/request_context", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/scala/com/twitter/timelines/prediction/common/aggregates", "src/scala/com/twitter/timelines/util", "src/thrift/com/twitter/ml/api:data-java", "src/thrift/com/twitter/timelines/suggests/common:data_record_metadata-scala", "src/thrift/com/twitter/timelines/suggests/common:dense_data_record-scala", "src/thrift/com/twitter/tweetypie:service-scala", "src/thrift/com/twitter/tweetypie:tweet-scala", "src/thrift/com/twitter/user_session_store:thrift-java", "stitch/stitch-core", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/repository/uss:versioned-aggregate-features-decoder", "timelines/data_processing/jobs/timeline_ranking_user_features:mini", "timelines/data_processing/ml_util/aggregation_framework:common_types", "timelines/data_processing/ml_util/aggregation_framework/conversion:for-timelines", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", "user_session_store/src/main/scala/com/twitter/user_session_store", "util/util-core", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseAggregateQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.repository.Repository import com.twitter.stitch.Stitch import com.twitter.timelines.aggregate_interactions.thriftjava.UserAggregateInteractions import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType.AggregateType import com.twitter.timelines.data_processing.ml_util.aggregation_framework.StoreConfig import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata import com.twitter.user_session_store.thriftjava.UserSession import com.twitter.util.Future abstract class BaseAggregateQueryFeatureHydrator( featureRepository: Repository[Long, Option[UserSession]], metadataRepository: Repository[Int, Option[DenseFeatureMetadata]], feature: Feature[PipelineQuery, Option[AggregateFeaturesToDecodeWithMetadata]]) extends QueryFeatureHydrator[PipelineQuery] { override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { val viewerId = query.getRequiredUserId Stitch.callFuture( featureRepository(viewerId) .flatMap { userSession: Option[UserSession] => val featuresWithMetadata: Option[Future[AggregateFeaturesToDecodeWithMetadata]] = userSession .flatMap(decodeUserSession(_)) featuresWithMetadata .map { fu: Future[AggregateFeaturesToDecodeWithMetadata] => fu.map(Some(_)) } .getOrElse(Future.None) .map { value => FeatureMapBuilder() .add(feature, value) .build() } } ) } private def decodeUserSession( session: UserSession ): Option[Future[AggregateFeaturesToDecodeWithMetadata]] = { Option(session.user_aggregate_interactions).flatMap { aggregates => aggregates.getSetField match { case UserAggregateInteractions._Fields.V17 => Some( getAggregateFeaturesWithMetadata( aggregates.getV17.user_aggregates.versionId, UserAggregateInteractions.v17(aggregates.getV17)) ) case _ => None } } } private def getAggregateFeaturesWithMetadata( versionId: Int, userAggregateInteractions: UserAggregateInteractions, ): Future[AggregateFeaturesToDecodeWithMetadata] = { metadataRepository(versionId) .map(AggregateFeaturesToDecodeWithMetadata(_, userAggregateInteractions)) } } trait BaseAggregateRootFeature extends Feature[PipelineQuery, Option[AggregateFeaturesToDecodeWithMetadata]] { def aggregateStores: Set[StoreConfig[_]] lazy val aggregateTypes: Set[AggregateType] = aggregateStores.map(_.aggregateType) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.IRecordOneToOneAdapter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType.AggregateType import com.twitter.timelines.suggests.common.dense_data_record.thriftjava.DenseCompactDataRecord import java.lang.{Long => JLong} import java.util.{Map => JMap} abstract case class BaseEdgeAggregateFeature( aggregateGroups: Set[AggregateGroup], aggregateType: AggregateType, extractMapFn: AggregateFeaturesToDecodeWithMetadata => JMap[JLong, DenseCompactDataRecord], adapter: IRecordOneToOneAdapter[Seq[DataRecord]], getSecondaryKeysFn: CandidateWithFeatures[TweetCandidate] => Seq[Long]) extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord private val rootFeatureInfo = new AggregateFeatureInfo(aggregateGroups, aggregateType) val featureContext: FeatureContext = rootFeatureInfo.featureContext val rootFeature: BaseAggregateRootFeature = rootFeatureInfo.feature } trait BaseEdgeAggregateFeatureHydrator extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { def aggregateFeatures: Set[BaseEdgeAggregateFeature] override def features = aggregateFeatures.asInstanceOf[Set[Feature[_, _]]] override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { val featureMapBuilders: Seq[FeatureMapBuilder] = for (_ <- candidates) yield FeatureMapBuilder() aggregateFeatures.foreach { feature => val featureValues = hydrateAggregateFeature(query, candidates, feature) (featureMapBuilders zip featureValues).foreach { case (featureMapBuilder, featureValue) => featureMapBuilder.add(feature, featureValue) } } featureMapBuilders.map(_.build()) } private def hydrateAggregateFeature( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]], feature: BaseEdgeAggregateFeature ): Seq[DataRecord] = { val rootFeature = feature.rootFeature val extractMapFn = feature.extractMapFn val featureContext = feature.featureContext val secondaryIds: Seq[Seq[Long]] = candidates.map(feature.getSecondaryKeysFn) val featuresToDecodeWithMetadata = query.features .flatMap(_.getOrElse(rootFeature, None)) .getOrElse(AggregateFeaturesToDecodeWithMetadata.empty) // Decode the DenseCompactDataRecords into DataRecords for each required secondary id. val decoded: Map[Long, DataRecord] = Utils.selectAndTransform( secondaryIds.flatten.distinct, featuresToDecodeWithMetadata.toDataRecord, extractMapFn(featuresToDecodeWithMetadata) ) // Remove unnecessary features in-place. This is safe because the underlying DataRecords // are unique and have just been generated in the previous step. decoded.values.foreach(Utils.filterDataRecord(_, featureContext)) // Put features into the FeatureMapBuilders secondaryIds.map { ids => val dataRecords = ids.flatMap(decoded.get) feature.adapter.adaptToDataRecord(dataRecords) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.offline_aggregates.PassThroughAdapter import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.offline_aggregates.SparseAggregatesToDenseAdapter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TSPInferredTopicFeature import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig.CombineCountPolicies object EdgeAggregateFeatures { object UserAuthorAggregateFeature extends BaseEdgeAggregateFeature( aggregateGroups = TimelinesAggregationConfig.userAuthorAggregatesV2 ++ Set( TimelinesAggregationConfig.userAuthorAggregatesV5, TimelinesAggregationConfig.tweetSourceUserAuthorAggregatesV1, TimelinesAggregationConfig.twitterWideUserAuthorAggregates ), aggregateType = AggregateType.UserAuthor, extractMapFn = _.userAuthorAggregates, adapter = PassThroughAdapter, getSecondaryKeysFn = _.features.getOrElse(AuthorIdFeature, None).toSeq ) object UserOriginalAuthorAggregateFeature extends BaseEdgeAggregateFeature( aggregateGroups = Set(TimelinesAggregationConfig.userOriginalAuthorAggregatesV1), aggregateType = AggregateType.UserOriginalAuthor, extractMapFn = _.userOriginalAuthorAggregates, adapter = PassThroughAdapter, getSecondaryKeysFn = candidate => CandidatesUtil.getOriginalAuthorId(candidate.features).toSeq ) object UserTopicAggregateFeature extends BaseEdgeAggregateFeature( aggregateGroups = Set( TimelinesAggregationConfig.userTopicAggregates, TimelinesAggregationConfig.userTopicAggregatesV2, ), aggregateType = AggregateType.UserTopic, extractMapFn = _.userTopicAggregates, adapter = PassThroughAdapter, getSecondaryKeysFn = candidate => candidate.features.getOrElse(TopicIdSocialContextFeature, None).toSeq ) object UserMentionAggregateFeature extends BaseEdgeAggregateFeature( aggregateGroups = Set(TimelinesAggregationConfig.userMentionAggregates), aggregateType = AggregateType.UserMention, extractMapFn = _.userMentionAggregates, adapter = new SparseAggregatesToDenseAdapter(CombineCountPolicies.MentionCountsPolicy), getSecondaryKeysFn = candidate => candidate.features.getOrElse(MentionScreenNameFeature, Seq.empty).map(_.hashCode.toLong) ) object UserInferredTopicAggregateFeature extends BaseEdgeAggregateFeature( aggregateGroups = Set( TimelinesAggregationConfig.userInferredTopicAggregates, ), aggregateType = AggregateType.UserInferredTopic, extractMapFn = _.userInferredTopicAggregates, adapter = new SparseAggregatesToDenseAdapter( CombineCountPolicies.UserInferredTopicCountsPolicy), getSecondaryKeysFn = candidate => candidate.features.getOrElse(TSPInferredTopicFeature, Map.empty[Long, Double]).keys.toSeq ) object UserInferredTopicAggregateV2Feature extends BaseEdgeAggregateFeature( aggregateGroups = Set( TimelinesAggregationConfig.userInferredTopicAggregatesV2 ), aggregateType = AggregateType.UserInferredTopic, extractMapFn = _.userInferredTopicAggregates, adapter = new SparseAggregatesToDenseAdapter( CombineCountPolicies.UserInferredTopicV2CountsPolicy), getSecondaryKeysFn = candidate => candidate.features.getOrElse(TSPInferredTopicFeature, Map.empty[Long, Double]).keys.toSeq ) object UserMediaUnderstandingAnnotationAggregateFeature extends BaseEdgeAggregateFeature( aggregateGroups = Set( TimelinesAggregationConfig.userMediaUnderstandingAnnotationAggregates), aggregateType = AggregateType.UserMediaUnderstandingAnnotation, extractMapFn = _.userMediaUnderstandingAnnotationAggregates, adapter = new SparseAggregatesToDenseAdapter( CombineCountPolicies.UserMediaUnderstandingAnnotationCountsPolicy), getSecondaryKeysFn = candidate => CandidatesUtil.getMediaUnderstandingAnnotationIds(candidate.features) ) object UserEngagerAggregateFeature extends BaseEdgeAggregateFeature( aggregateGroups = Set(TimelinesAggregationConfig.userEngagerAggregates), aggregateType = AggregateType.UserEngager, extractMapFn = _.userEngagerAggregates, adapter = new SparseAggregatesToDenseAdapter(CombineCountPolicies.EngagerCountsPolicy), getSecondaryKeysFn = candidate => CandidatesUtil.getEngagerUserIds(candidate.features) ) object UserEngagerGoodClickAggregateFeature extends BaseEdgeAggregateFeature( aggregateGroups = Set(TimelinesAggregationConfig.userEngagerGoodClickAggregates), aggregateType = AggregateType.UserEngager, extractMapFn = _.userEngagerAggregates, adapter = new SparseAggregatesToDenseAdapter( CombineCountPolicies.EngagerGoodClickCountsPolicy), getSecondaryKeysFn = candidate => CandidatesUtil.getEngagerUserIds(candidate.features) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregateMetadataRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregatePartARepository import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.servo.repository.Repository import com.twitter.timelines.data_processing.jobs.timeline_ranking_user_features.TimelinesPartAStoreRegister import com.twitter.timelines.data_processing.ml_util.aggregation_framework.StoreConfig import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata import com.twitter.user_session_store.thriftjava.UserSession import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton object PartAAggregateRootFeature extends BaseAggregateRootFeature { override val aggregateStores: Set[StoreConfig[_]] = TimelinesPartAStoreRegister.allStores } @Singleton class PartAAggregateQueryFeatureHydrator @Inject() ( @Named(TimelineAggregatePartARepository) repository: Repository[Long, Option[UserSession]], @Named(TimelineAggregateMetadataRepository) metadataRepository: Repository[Int, Option[DenseFeatureMetadata]]) extends BaseAggregateQueryFeatureHydrator( repository, metadataRepository, PartAAggregateRootFeature ) { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PartAAggregateQuery") override val features = Set(PartAAggregateRootFeature) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregateMetadataRepository import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregatePartBRepository import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecordMerger import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.repository.Repository import com.twitter.stitch.Stitch import com.twitter.timelines.data_processing.jobs.timeline_ranking_user_features.TimelinesPartBStoreRegister import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType import com.twitter.timelines.data_processing.ml_util.aggregation_framework.StoreConfig import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata import com.twitter.user_session_store.thriftjava.UserSession import com.twitter.util.Time import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton object PartBAggregateRootFeature extends BaseAggregateRootFeature { override val aggregateStores: Set[StoreConfig[_]] = TimelinesPartBStoreRegister.allStores } object UserAggregateFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class PartBAggregateQueryFeatureHydrator @Inject() ( @Named(TimelineAggregatePartBRepository) repository: Repository[Long, Option[UserSession]], @Named(TimelineAggregateMetadataRepository) metadataRepository: Repository[Int, Option[DenseFeatureMetadata]]) extends BaseAggregateQueryFeatureHydrator( repository, metadataRepository, PartBAggregateRootFeature ) { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PartBAggregateQuery") override val features: Set[Feature[_, _]] = Set(PartBAggregateRootFeature, UserAggregateFeature) private val userAggregateFeatureInfo = new AggregateFeatureInfo( aggregateGroups = Set( TimelinesAggregationConfig.userAggregatesV2, TimelinesAggregationConfig.userAggregatesV5Continuous, TimelinesAggregationConfig.userAggregatesV6, TimelinesAggregationConfig.twitterWideUserAggregates, ), aggregateType = AggregateType.User ) private val userHourAggregateFeatureInfo = new AggregateFeatureInfo( aggregateGroups = Set( TimelinesAggregationConfig.userRequestHourAggregates, ), aggregateType = AggregateType.UserRequestHour ) private val userDowAggregateFeatureInfo = new AggregateFeatureInfo( aggregateGroups = Set( TimelinesAggregationConfig.userRequestDowAggregates ), aggregateType = AggregateType.UserRequestDow ) require( userAggregateFeatureInfo.feature == PartBAggregateRootFeature, "UserAggregates feature must be provided by the PartB data source.") require( userHourAggregateFeatureInfo.feature == PartBAggregateRootFeature, "UserRequstHourAggregates feature must be provided by the PartB data source.") require( userDowAggregateFeatureInfo.feature == PartBAggregateRootFeature, "UserRequestDowAggregates feature must be provided by the PartB data source.") override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { // Hydrate TimelineAggregatePartBFeature and UserAggregateFeature sequentially. super.hydrate(query).map { featureMap => val time: Time = Time.now val hourOfDay = RequestContextAdapter.hourFromTimestamp(time.inMilliseconds) val dayOfWeek = RequestContextAdapter.dowFromTimestamp(time.inMilliseconds) val dr = featureMap .get(PartBAggregateRootFeature).map { featuresWithMetadata => val userAggregatesDr = featuresWithMetadata.userAggregatesOpt .map(featuresWithMetadata.toDataRecord) val userRequestHourAggregatesDr = Option(featuresWithMetadata.userRequestHourAggregates.get(hourOfDay)) .map(featuresWithMetadata.toDataRecord) val userRequestDowAggregatesDr = Option(featuresWithMetadata.userRequestDowAggregates.get(dayOfWeek)) .map(featuresWithMetadata.toDataRecord) dropUnknownFeatures(userAggregatesDr, userAggregateFeatureInfo.featureContext) dropUnknownFeatures( userRequestHourAggregatesDr, userHourAggregateFeatureInfo.featureContext) dropUnknownFeatures( userRequestDowAggregatesDr, userDowAggregateFeatureInfo.featureContext) mergeDataRecordOpts( userAggregatesDr, userRequestHourAggregatesDr, userRequestDowAggregatesDr) }.getOrElse(new DataRecord()) featureMap + (UserAggregateFeature, dr) } } private val drMerger = new DataRecordMerger private def mergeDataRecordOpts(dataRecordOpts: Option[DataRecord]*): DataRecord = dataRecordOpts.flatten.foldLeft(new DataRecord) { (l, r) => drMerger.merge(l, r) l } private def dropUnknownFeatures( dataRecordOpt: Option[DataRecord], featureContext: FeatureContext ): Unit = dataRecordOpt.foreach(new RichDataRecord(_, featureContext).dropUnknownFeatures()) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase1EdgeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures._ import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import javax.inject.Inject import javax.inject.Singleton @Singleton class Phase1EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Phase1EdgeAggregate") override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = Set( UserAuthorAggregateFeature, UserOriginalAuthorAggregateFeature, UserMentionAggregateFeature ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserEngagerAggregateFeature import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserEngagerGoodClickAggregateFeature import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserInferredTopicAggregateFeature import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserInferredTopicAggregateV2Feature import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserMediaUnderstandingAnnotationAggregateFeature import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserTopicAggregateFeature import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import javax.inject.Inject import javax.inject.Singleton @Singleton class Phase2EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Phase2EdgeAggregate") override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = Set( UserEngagerAggregateFeature, UserEngagerGoodClickAggregateFeature, UserInferredTopicAggregateFeature, UserInferredTopicAggregateV2Feature, UserTopicAggregateFeature, UserMediaUnderstandingAnnotationAggregateFeature ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Utils.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.RichDataRecord import com.twitter.timelines.suggests.common.dense_data_record.thriftjava.DenseCompactDataRecord private[offline_aggregates] object Utils { /** * Selects only those values in map that correspond to the keys in ids and apply the provided * transform to the selected values. This is a convenience method for use by Timelines Aggregation * Framework based features. * * @param idsToSelect The set of ids to extract values for. * @param transform A transform to apply to the selected values. * @param map Map[Long, DenseCompactDataRecord] */ def selectAndTransform( idsToSelect: Seq[Long], transform: DenseCompactDataRecord => DataRecord, map: java.util.Map[java.lang.Long, DenseCompactDataRecord], ): Map[Long, DataRecord] = { val filtered: Seq[(Long, DataRecord)] = for { id <- idsToSelect if map.containsKey(id) } yield { id -> transform(map.get(id)) } filtered.toMap } def filterDataRecord(dr: DataRecord, featureContext: FeatureContext): Unit = { new RichDataRecord(dr, featureContext).dropUnknownFeatures() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "finatra/inject/inject-core/src/main/scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "servo/repo/src/main/scala", "src/java/com/twitter/ml/api:api-base", "src/java/com/twitter/ml/api/constant", "src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/timelines/prediction/common/aggregates/real_time:base-config", "src/thrift/com/twitter/ml/api:data-java", "src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-java", "stitch/stitch-core", "timelines/data_processing/ml_util/aggregation_framework:common_types", "timelines/data_processing/ml_util/aggregation_framework/heron", "util/util-core", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateBulkCandidateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch trait BaseRealTimeAggregateBulkCandidateFeatureHydrator[K] extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] with BaseRealtimeAggregateHydrator[K] { val outputFeature: DataRecordInAFeature[TweetCandidate] override def features: Set[Feature[_, _]] = Set(outputFeature) override lazy val statScope: String = identifier.toString def keysFromQueryAndCandidates( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[K]] override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { val possiblyKeys = keysFromQueryAndCandidates(query, candidates) fetchAndConstructDataRecords(possiblyKeys).map { dataRecords => dataRecords.map { dataRecord => FeatureMapBuilder().add(outputFeature, dataRecord).build() } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateQueryFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch trait BaseRealTimeAggregateQueryFeatureHydrator[K] extends QueryFeatureHydrator[PipelineQuery] with BaseRealtimeAggregateHydrator[K] { val outputFeature: DataRecordInAFeature[PipelineQuery] override def features: Set[Feature[_, _]] = Set(outputFeature) override lazy val statScope: String = identifier.toString def keysFromQueryAndCandidates( query: PipelineQuery ): Option[K] override def hydrate( query: PipelineQuery ): Stitch[FeatureMap] = OffloadFuturePools.offloadFuture { val possiblyKeys = keysFromQueryAndCandidates(query) fetchAndConstructDataRecords(Seq(possiblyKeys)).map { dataRecords => FeatureMapBuilder() .add(outputFeature, dataRecords.head) .build() } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealtimeAggregateHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates.BaseRealtimeAggregateHydrator._ import com.twitter.home_mixer.util.DataRecordUtil import com.twitter.home_mixer.util.ObservedKeyValueResultHandler import com.twitter.ml.api.DataRecord import com.twitter.ml.api.DataRecordMerger import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.constant.SharedFeatures import com.twitter.ml.api.util.SRichDataRecord import com.twitter.ml.api.{Feature => MLApiFeature} import com.twitter.servo.cache.ReadCache import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.util.Future import com.twitter.util.Time import com.twitter.util.Try import java.lang.{Double => JDouble} import scala.collection.JavaConverters._ trait BaseRealtimeAggregateHydrator[K] extends ObservedKeyValueResultHandler { val client: ReadCache[K, DataRecord] val aggregateGroups: Seq[AggregateGroup] val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map.empty private lazy val typedAggregateGroupsList = aggregateGroups.map(_.buildTypedAggregateGroups()) private lazy val featureContexts: Seq[FeatureContext] = typedAggregateGroupsList.map { typedAggregateGroups => new FeatureContext( (SharedFeatures.TIMESTAMP +: typedAggregateGroups.flatMap(_.allOutputFeatures)).asJava ) } private lazy val aggregateFeaturesRenameMap: Map[MLApiFeature[_], MLApiFeature[_]] = { val prefixes: Seq[Option[String]] = aggregateGroups.map(aggregateGroupToPrefix.get) typedAggregateGroupsList .zip(prefixes).map { case (typedAggregateGroups, prefix) => if (prefix.nonEmpty) typedAggregateGroups .map { _.outputFeaturesToRenamedOutputFeatures(prefix.get) }.reduce(_ ++ _) else Map.empty[MLApiFeature[_], MLApiFeature[_]] }.reduce(_ ++ _) } private lazy val renamedFeatureContexts: Seq[FeatureContext] = typedAggregateGroupsList.map { typedAggregateGroups => val renamedAllOutputFeatures = typedAggregateGroups.flatMap(_.allOutputFeatures).map { feature => aggregateFeaturesRenameMap.getOrElse(feature, feature) } new FeatureContext(renamedAllOutputFeatures.asJava) } private lazy val decays: Seq[TimeDecay] = typedAggregateGroupsList.map { typedAggregateGroups => RealTimeAggregateTimeDecay( typedAggregateGroups.flatMap(_.continuousFeatureIdsToHalfLives).toMap) .apply(_, _) } private val drMerger = new DataRecordMerger private def postTransformer(dataRecord: Try[Option[DataRecord]]): Try[DataRecord] = { dataRecord.map { case Some(dr) => val newDr = new DataRecord() featureContexts.zip(renamedFeatureContexts).zip(decays).foreach { case ((featureContext, renamedFeatureContext), decay) => val decayedDr = applyDecay(dr, featureContext, decay) val renamedDr = DataRecordUtil.applyRename( dataRecord = decayedDr, featureContext, renamedFeatureContext, aggregateFeaturesRenameMap) drMerger.merge(newDr, renamedDr) } newDr case _ => new DataRecord } } def fetchAndConstructDataRecords(possiblyKeys: Seq[Option[K]]): Future[Seq[Try[DataRecord]]] = { val keys = possiblyKeys.flatten val response: Future[KeyValueResult[K, DataRecord]] = if (keys.isEmpty) Future.value(KeyValueResult.empty) else { val batchResponses = keys .grouped(RequestBatchSize) .map(keyGroup => client.get(keyGroup)) .toSeq Future.collect(batchResponses).map(_.reduce(_ ++ _)) } response.map { result => possiblyKeys.map { possiblyKey => val value = observedGet(key = possiblyKey, keyValueResult = result) postTransformer(value) } } } } object BaseRealtimeAggregateHydrator { private val RequestBatchSize = 5 type TimeDecay = scala.Function2[com.twitter.ml.api.DataRecord, scala.Long, scala.Unit] private def applyDecay( dataRecord: DataRecord, featureContext: FeatureContext, decay: TimeDecay ): DataRecord = { def time: Long = Time.now.inMillis val richFullDr = new SRichDataRecord(dataRecord, featureContext) val richNewDr = new SRichDataRecord(new DataRecord, featureContext) val featureIterator = featureContext.iterator() featureIterator.forEachRemaining { feature => if (richFullDr.hasFeature(feature)) { val typedFeature = feature.asInstanceOf[MLApiFeature[JDouble]] richNewDr.setFeatureValue(typedFeature, richFullDr.getFeatureValue(typedFeature)) } } val resultDr = richNewDr.getRecord decay(resultDr, time) resultDr } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.EngagementsReceivedByAuthorCache import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.ReadCache import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import javax.inject.Inject import javax.inject.Singleton object EngagementsReceivedByAuthorRealTimeAggregateFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator @Inject() ( @Named(EngagementsReceivedByAuthorCache) override val client: ReadCache[Long, DataRecord], override val statsReceiver: StatsReceiver) extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("EngagementsReceivedByAuthorRealTimeAggregate") override val outputFeature: DataRecordInAFeature[TweetCandidate] = EngagementsReceivedByAuthorRealTimeAggregateFeature override val aggregateGroups: Seq[AggregateGroup] = Seq( authorEngagementRealTimeAggregatesProd, authorShareEngagementsRealTimeAggregates ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( authorShareEngagementsRealTimeAggregates -> "original_author.timelines.author_share_engagements_real_time_aggregates." ) override def keysFromQueryAndCandidates( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[Long]] = candidates.map(candidate => CandidatesUtil.getOriginalAuthorId(candidate.features)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/RealTimeAggregateTimeDecay.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.twitter.ml.api.DataRecord import com.twitter.ml.api.constant.SharedFeatures.TIMESTAMP import com.twitter.util.Duration /** * The default TimeDecay implementation for real time aggregates. * * @param featureIdToHalfLife A precomputed map from aggregate feature ids to their half lives. * @param timestampFeatureId A discrete timestamp feature id. */ case class RealTimeAggregateTimeDecay( featureIdToHalfLife: Map[Long, Duration], timestampFeatureId: Long = TIMESTAMP.getFeatureId) { /** * Mutates the data record which is just a reference to the input. * * @param record Data record to apply decay to (is mutated). * @param timeNow The current read time (in milliseconds) to decay counts forward to. */ def apply(record: DataRecord, timeNow: Long): Unit = { if (record.isSetDiscreteFeatures) { val discreteFeatures = record.getDiscreteFeatures if (discreteFeatures.containsKey(timestampFeatureId)) { if (record.isSetContinuousFeatures) { val ctsFeatures = record.getContinuousFeatures val storedTimestamp: Long = discreteFeatures.get(timestampFeatureId) val scaledDt = if (timeNow > storedTimestamp) { (timeNow - storedTimestamp).toDouble * math.log(2) } else 0.0 featureIdToHalfLife.foreach { case (featureId, halfLife) => if (ctsFeatures.containsKey(featureId)) { val storedValue = ctsFeatures.get(featureId) val alpha = if (halfLife.inMilliseconds != 0) math.exp(-scaledDt / halfLife.inMilliseconds) else 0 val decayedValue: Double = alpha * storedValue record.putToContinuousFeatures(featureId, decayedValue) } } } discreteFeatures.remove(timestampFeatureId) } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicCountryEngagementCache import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.ReadCache import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import javax.inject.Inject import javax.inject.Singleton object TopicCountryEngagementRealTimeAggregateFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TopicCountryEngagementRealTimeAggregateFeatureHydrator @Inject() ( @Named(TopicCountryEngagementCache) override val client: ReadCache[(Long, String), DataRecord], override val statsReceiver: StatsReceiver) extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, String)] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TopicCountryEngagementRealTimeAggregate") override val outputFeature: DataRecordInAFeature[TweetCandidate] = TopicCountryEngagementRealTimeAggregateFeature override val aggregateGroups: Seq[AggregateGroup] = Seq( topicCountryRealTimeAggregates ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( topicCountryRealTimeAggregates -> "topic-country_code.timelines.topic_country_engagement_real_time_aggregates." ) override def keysFromQueryAndCandidates( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[(Long, String)]] = { candidates.map { candidate => val maybeTopicId = candidate.features .getTry(TopicIdSocialContextFeature) .toOption .flatten val maybeCountryCode = query.clientContext.countryCode for { topicId <- maybeTopicId countryCode <- maybeCountryCode } yield (topicId, countryCode) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicEngagementCache import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.ReadCache import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton object TopicEngagementRealTimeAggregateFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TopicEngagementRealTimeAggregateFeatureHydrator @Inject() ( @Named(TopicEngagementCache) override val client: ReadCache[Long, DataRecord], override val statsReceiver: StatsReceiver) extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TopicEngagementRealTimeAggregate") override val outputFeature: DataRecordInAFeature[TweetCandidate] = TopicEngagementRealTimeAggregateFeature override val aggregateGroups: Seq[AggregateGroup] = Seq( topicEngagementRealTimeAggregatesProd, topicEngagement24HourRealTimeAggregatesProd, topicShareEngagementsRealTimeAggregates ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( topicEngagement24HourRealTimeAggregatesProd -> "topic.timelines.topic_engagement_24_hour_real_time_aggregates.", topicShareEngagementsRealTimeAggregates -> "topic.timelines.topic_share_engagements_real_time_aggregates." ) override def keysFromQueryAndCandidates( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[Long]] = { candidates.map { candidate => candidate.features .getTry(TopicIdSocialContextFeature) .toOption .flatten } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TweetCountryEngagementRealTimeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetCountryEngagementCache import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.ReadCache import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import javax.inject.Inject import javax.inject.Singleton object TweetCountryEngagementRealTimeAggregateFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TweetCountryEngagementRealTimeAggregateFeatureHydrator @Inject() ( @Named(TweetCountryEngagementCache) override val client: ReadCache[(Long, String), DataRecord], override val statsReceiver: StatsReceiver) extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, String)] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetCountryEngagementRealTimeAggregate") override val outputFeature: DataRecordInAFeature[TweetCandidate] = TweetCountryEngagementRealTimeAggregateFeature override val aggregateGroups: Seq[AggregateGroup] = Seq( tweetCountryRealTimeAggregates, tweetCountryPrivateEngagementsRealTimeAggregates ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( tweetCountryRealTimeAggregates -> "tweet-country_code.timelines.tweet_country_engagement_real_time_aggregates.", tweetCountryPrivateEngagementsRealTimeAggregates -> "tweet-country_code.timelines.tweet_country_private_engagement_real_time_aggregates." ) override def keysFromQueryAndCandidates( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[(Long, String)]] = { val countryCode = query.clientContext.countryCode candidates.map { candidate => val originalTweetId = CandidatesUtil.getOriginalTweetId(candidate) countryCode.map((originalTweetId, _)) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TweetEngagementRealTimeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetEngagementCache import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.ReadCache import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import javax.inject.Inject import javax.inject.Singleton object TweetEngagementRealTimeAggregateFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TweetEngagementRealTimeAggregateFeatureHydrator @Inject() ( @Named(TweetEngagementCache) override val client: ReadCache[Long, DataRecord], override val statsReceiver: StatsReceiver) extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetEngagementRealTimeAggregate") override val outputFeature: DataRecordInAFeature[TweetCandidate] = TweetEngagementRealTimeAggregateFeature override val aggregateGroups: Seq[AggregateGroup] = Seq( tweetEngagement30MinuteCountsProd, tweetEngagementTotalCountsProd, tweetEngagementUserStateRealTimeAggregatesProd, tweetNegativeEngagementUserStateRealTimeAggregates, tweetNegativeEngagement6HourCounts, tweetNegativeEngagementTotalCounts, tweetShareEngagementsRealTimeAggregates, tweetBCEDwellEngagementsRealTimeAggregates ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( tweetShareEngagementsRealTimeAggregates -> "original_tweet.timelines.tweet_share_engagements_real_time_aggregates.", tweetBCEDwellEngagementsRealTimeAggregates -> "original_tweet.timelines.tweet_bce_dwell_engagements_real_time_aggregates." ) override def keysFromQueryAndCandidates( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[Long]] = { candidates .map(candidate => Some(CandidatesUtil.getOriginalTweetId(candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TwitterListEngagementRealTimeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.TwitterListIdFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwitterListEngagementCache import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.ReadCache import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import javax.inject.Inject import javax.inject.Singleton object TwitterListEngagementRealTimeAggregateFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class TwitterListEngagementRealTimeAggregateFeatureHydrator @Inject() ( @Named(TwitterListEngagementCache) override val client: ReadCache[Long, DataRecord], override val statsReceiver: StatsReceiver) extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TwitterListEngagementRealTimeAggregate") override val outputFeature: DataRecordInAFeature[TweetCandidate] = TwitterListEngagementRealTimeAggregateFeature override val aggregateGroups: Seq[AggregateGroup] = Seq( listEngagementRealTimeAggregatesProd ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( listEngagementRealTimeAggregatesProd -> "twitter_list.timelines.twitter_list_engagement_real_time_aggregates." ) override def keysFromQueryAndCandidates( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[Long]] = { candidates.map { candidate => candidate.features .getTry(TwitterListIdFeature) .toOption .flatten } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/UserAuthorEngagementRealTimeAggregateFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserAuthorEngagementCache import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.ReadCache import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import javax.inject.Inject import javax.inject.Singleton object UserAuthorEngagementRealTimeAggregateFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class UserAuthorEngagementRealTimeAggregateFeatureHydrator @Inject() ( @Named(UserAuthorEngagementCache) override val client: ReadCache[(Long, Long), DataRecord], override val statsReceiver: StatsReceiver) extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, Long)] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserAuthorEngagementRealTimeAggregate") override val outputFeature: DataRecordInAFeature[TweetCandidate] = UserAuthorEngagementRealTimeAggregateFeature override val aggregateGroups: Seq[AggregateGroup] = Seq( userAuthorEngagementRealTimeAggregatesProd, userAuthorShareEngagementsRealTimeAggregates ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( userAuthorEngagementRealTimeAggregatesProd -> "user-author.timelines.user_author_engagement_real_time_aggregates.", userAuthorShareEngagementsRealTimeAggregates -> "user-author.timelines.user_author_share_engagements_real_time_aggregates." ) override def keysFromQueryAndCandidates( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[Option[(Long, Long)]] = { val userId = query.getRequiredUserId candidates.map { candidate => CandidatesUtil .getOriginalAuthorId(candidate.features) .map((userId, _)) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/UserEngagementRealTimeAggregatesFeatureHydrator.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates import com.google.inject.name.Named import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserEngagementCache import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.ReadCache import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ import javax.inject.Inject import javax.inject.Singleton object UserEngagementRealTimeAggregateFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton class UserEngagementRealTimeAggregatesFeatureHydrator @Inject() ( @Named(UserEngagementCache) override val client: ReadCache[Long, DataRecord], override val statsReceiver: StatsReceiver) extends BaseRealTimeAggregateQueryFeatureHydrator[Long] { override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserEngagementRealTimeAggregates") override val outputFeature: DataRecordInAFeature[PipelineQuery] = UserEngagementRealTimeAggregateFeature val aggregateGroups: Seq[AggregateGroup] = Seq( userEngagementRealTimeAggregatesProd, userShareEngagementsRealTimeAggregates, userBCEDwellEngagementsRealTimeAggregates, userEngagement48HourRealTimeAggregatesProd, userNegativeEngagementAuthorUserState72HourRealTimeAggregates, userNegativeEngagementAuthorUserStateRealTimeAggregates, userProfileEngagementRealTimeAggregates, ) override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( userShareEngagementsRealTimeAggregates -> "user.timelines.user_share_engagements_real_time_aggregates.", userBCEDwellEngagementsRealTimeAggregates -> "user.timelines.user_bce_dwell_engagements_real_time_aggregates.", userEngagement48HourRealTimeAggregatesProd -> "user.timelines.user_engagement_48_hour_real_time_aggregates.", userNegativeEngagementAuthorUserState72HourRealTimeAggregates -> "user.timelines.user_negative_engagement_author_user_state_72_hour_real_time_aggregates.", userProfileEngagementRealTimeAggregates -> "user.timelines.user_profile_engagement_real_time_aggregates." ) override def keysFromQueryAndCandidates(query: PipelineQuery): Option[Long] = { Some(query.getRequiredUserId) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/control_ai", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", "snowflake/src/main/scala/com/twitter/snowflake/id", "src/thrift/com/twitter/timelines/control_ai:timeline-control-ai-thrift-scala", "stitch/stitch-socialgraph", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ControlAiExcludeFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ControlAiEmbeddingSimilarityThresholdParam import com.twitter.home_mixer.product.scored_tweets.util.ControlAiUtil import com.twitter.product_mixer.component_library.feature_hydrator.query.control_ai.ControlAiTopicEmbeddingMapFeature import com.twitter.product_mixer.component_library.feature_hydrator.query.control_ai.UserControlAiFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.control_ai.control.{thriftscala => ci} object ControlAiExcludeFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("ControlAiExclude") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val actions = query.features .flatMap(_.getOrElse(UserControlAiFeature, None)) .map(_.actions).getOrElse(Seq.empty).filter(_.actionType == ci.ActionType.Exclude) val (removed, kept) = candidates.partition { candidate => actions.exists( ControlAiUtil.conditionMatch( _, candidate, query.features .map(_.get(ControlAiTopicEmbeddingMapFeature)).getOrElse(Map.empty), threshold = query.params(ControlAiEmbeddingSimilarityThresholdParam) )) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ControlAiOnlyIncludeFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ControlAiEmbeddingSimilarityThresholdParam import com.twitter.home_mixer.product.scored_tweets.util.ControlAiUtil import com.twitter.product_mixer.component_library.feature_hydrator.query.control_ai.ControlAiTopicEmbeddingMapFeature import com.twitter.product_mixer.component_library.feature_hydrator.query.control_ai.UserControlAiFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.control_ai.control.{thriftscala => ci} object ControlAiOnlyIncludeFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("ControlAiOnlyInclude") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val actions = query.features .flatMap(_.getOrElse(UserControlAiFeature, None)) .map(_.actions).getOrElse(Seq.empty).filter(_.actionType == ci.ActionType.Only) val (kept, removed) = candidates.partition { candidate => actions.isEmpty || actions.exists( ControlAiUtil.conditionMatch( _, candidate, query.features .map(_.get(ControlAiTopicEmbeddingMapFeature)).getOrElse(Map.empty), threshold = query.params(ControlAiEmbeddingSimilarityThresholdParam) )) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/CustomSnowflakeIdAgeFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.Param import com.twitter.util.Duration /** * Filters Snowflake ID-compatible content that are older than some configurable threshold. * * We derive the content age from the Snowflake ID (http://go/snowflake), and non-Snowflake IDs * are assumed to be too old. * * @param maxAgeParam Feature Switch configurable for convenience * @tparam Candidate The type of the candidates */ case class CustomSnowflakeIdAgeFilter[Candidate <: UniversalNoun[Long]]( maxAgeParam: Param[Duration]) extends Filter[PipelineQuery, Candidate] { override val identifier: FilterIdentifier = FilterIdentifier("SnowflakeIdAge") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[Candidate]] ): Stitch[FilterResult[Candidate]] = { val maxAge = query.params(maxAgeParam) val (keptCandidates, removedCandidates) = candidates .map(c => (c.candidate, c.features.get(CandidatePipelines))) .partition { filterCandidate => SnowflakeId.timeFromIdOpt(filterCandidate._1.id) match { case Some(creationTime) => query.queryTime.since(creationTime) <= maxAge case _ => false // Always deny if non-Snowflake } } Stitch.value( FilterResult(kept = keptCandidates.map(_._1), removed = removedCandidates.map(_._1))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/CustomSnowflakeIdAgeFilterWithIncludeRule.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.Param import com.twitter.util.Duration /** * Filters Snowflake ID-compatible content that are older than some configurable threshold, * but only for those candidates that match the includeRule. * * Candidates not matching the includeRule will always be kept. * * @param maxAgeParam Feature Switch configurable for convenience * @tparam Candidate The type of the candidates */ case class CustomSnowflakeIdAgeFilterWithIncludeRule[Candidate <: UniversalNoun[Long]]( maxAgeParam: Param[Duration], includeRule: Option[CandidateWithFeatures[Candidate] => Boolean] = None) extends Filter[PipelineQuery, Candidate] { override val identifier: FilterIdentifier = FilterIdentifier("SnowflakeIdAgeWithIncludeRule") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[Candidate]] ): Stitch[FilterResult[Candidate]] = { val maxAge = query.params(maxAgeParam) val (keptCandidates, removedCandidates) = candidates.partition { filterCandidate => includeRule match { case Some(rule) if rule(filterCandidate) => SnowflakeId.timeFromIdOpt(filterCandidate.candidate.id) match { case Some(creationTime) => query.queryTime.since(creationTime) <= maxAge case _ => false } case _ => true } } Stitch.value( FilterResult( kept = keptCandidates.map(_.candidate), removed = removedCandidates.map(_.candidate) ) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/CustomSnowflakeIdAgeFilterWithSkipRule.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.snowflake.id.SnowflakeId import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.Param import com.twitter.util.Duration /** * Filters Snowflake ID-compatible content that are older than some configurable threshold. * * We derive the content age from the Snowflake ID (http://go/snowflake), and non-Snowflake IDs * are assumed to be too old. * * @param maxAgeParam Feature Switch configurable for convenience * @tparam Candidate The type of the candidates */ case class CustomSnowflakeIdAgeFilterWithSkipRule[Candidate <: UniversalNoun[Long]]( maxAgeParam: Param[Duration], skipRule: Option[CandidateWithFeatures[Candidate] => Boolean] = None) extends Filter[PipelineQuery, Candidate] { override val identifier: FilterIdentifier = FilterIdentifier("SnowflakeIdAgeWithSkipRule") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[Candidate]] ): Stitch[FilterResult[Candidate]] = { val maxAge = query.params(maxAgeParam) val (keptCandidates, removedCandidates) = candidates.partition { filterCandidate => skipRule match { case Some(rule) if rule(filterCandidate) => true case _ => SnowflakeId.timeFromIdOpt(filterCandidate.candidate.id) match { case Some(creationTime) => query.queryTime.since(creationTime) <= maxAge case _ => false } } } Stitch.value( FilterResult( kept = keptCandidates.map(_.candidate), removed = removedCandidates.map(_.candidate) ) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/DuplicateConversationTweetsFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Remove any candidate that is in the ancestor list of any reply, including retweets of ancestors. * Then deduplicate any replies with the same root tweet by score. * * E.g. if B replied to A and D was a retweet of A, we would prefer to drop D since otherwise * we may end up serving the same tweet twice in the timeline (e.g. serving both A->B and D). */ object DuplicateConversationTweetsFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("DuplicateConversationTweets") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val allAncestors = candidates .flatMap(_.features.getOrElse(AncestorsFeature, Seq.empty)) .map(_.tweetId).toSet val dedupedCandidates = candidates .filter(candidate => !allAncestors.contains(CandidatesUtil.getOriginalTweetId(candidate))) .groupBy(_.features.getOrElse(AncestorsFeature, Seq.empty).lastOption.map(_.tweetId)) .flatMap { case (Some(_), conversationCandidates) => Seq(conversationCandidates.maxBy(_.features.getOrElse(ScoreFeature, None)).candidate.id) case (None, nonConversationCandidates) => nonConversationCandidates.map(_.candidate.id) }.toSet val (kept, removed) = candidates.partition(candidate => dedupedCandidates.contains(candidate.candidate.id)) Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ExtendedDirectedAtFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object ExtendedDirectedAtFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("ExtendedDirectedAt") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val sgsFollowedUsers = query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).toSet.flatten val (removed, kept) = candidates.partition { candidate => val authorId = candidate.features.getOrElse(AuthorIdFeature, None) val inReplyToUser = candidate.features.getOrElse(InReplyToUserIdFeature, None) val directedAtUser = candidate.features.getOrElse(DirectedAtUserIdFeature, None) inReplyToUser.isEmpty && directedAtUser.exists(!sgsFollowedUsers.contains(_)) && authorId.exists(sgsFollowedUsers.contains) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/FollowedAuthorFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * This filter removes tweets who's author is in the follow list */ object FollowedAuthorFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("FollowedAuthor") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val sgsFollowedUsers = query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).toSet.flatten val (removed, kept) = candidates.partition( _.features.getOrElse(AuthorIdFeature, None).exists(sgsFollowedUsers.contains)) Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/GrokAutoTranslateLanguageFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.GrokTranslatedPostIsCachedFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFromTweetypieFeature import com.twitter.home_mixer.model.HomeFeatures.UserUnderstandableLanguagesFeature import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableGrokAutoTranslateLanguageFilter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.stitch.Stitch object GrokAutoTranslateLanguageFilter extends Filter[ScoredTweetsQuery, TweetCandidate] with Filter.Conditionally[ScoredTweetsQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("GrokAutoTranslateLanguage") override def onlyIf( query: ScoredTweetsQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Boolean = query.params(EnableGrokAutoTranslateLanguageFilter) override def apply( query: ScoredTweetsQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val allUserUnderstandableLanguages: Seq[String] = query.features .getOrElse(FeatureMap.empty).getOrElse( UserUnderstandableLanguagesFeature, Seq.empty[String]) val (kept, removed) = { if (!allUserUnderstandableLanguages.isEmpty) { candidates.partition { candidate => val inNetwork = candidate.features.getOrElse(InNetworkFeature, true) val postLanguageOpt = candidate.features.getOrElse(TweetLanguageFromTweetypieFeature, None).map(_.toLowerCase) val grokTranslateCacheExists = candidate.features.getOrElse(GrokTranslatedPostIsCachedFeature, true) postLanguageOpt.forall { postLanguage => inNetwork || grokTranslateCacheExists || allUserUnderstandableLanguages.contains(postLanguage) } } } else { (candidates, Seq.empty) } } val filterResult = FilterResult( kept = kept.map(_.candidate), removed = removed.map(_.candidate) ) Stitch.value(filterResult) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/IsOutOfNetworkColdStartPostFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.IsColdStartPostFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object IsOutOfNetworkColdStartPostFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("IsOutOfNetworkColdStartPost") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (removed, kept) = candidates.partition { candidate => val isColdStartPost = candidate.features.getOrElse( IsColdStartPostFeature, false ) // Id none, assume its not cold start, don't filter out val isOutOfNetwork = !candidate.features .getOrElse(InNetworkFeature, true) // If none, assume its in-network, don't filter out isColdStartPost && isOutOfNetwork } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/LanguageFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.functional_component.feature_hydrator.UserLanguagesFeature import com.twitter.home_mixer.model.HomeFeatures.DeviceLanguageFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFromTweetypieFeature import com.twitter.home_mixer.model.HomeFeatures.UserEngagedLanguagesFeature import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableLanguageFilter import com.twitter.home_mixer.util.LanguageCode.AllowedLanguageCodes import com.twitter.home_mixer.util.LanguageCode.languageToISO import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.stitch.Stitch object LanguageFilter extends Filter[ScoredTweetsQuery, TweetCandidate] with Filter.Conditionally[ScoredTweetsQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("Language") override def onlyIf( query: ScoredTweetsQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Boolean = query.params(EnableLanguageFilter) override def apply( query: ScoredTweetsQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val userLanguages = query.features.get .getOrElse(UserLanguagesFeature, Seq.empty) .flatMap(lang => languageToISO.get(lang.toString.toLowerCase)) .toSet val userEngagedLanguages = query.features.get .getOrElse(UserEngagedLanguagesFeature, Set.empty[String]) .map(_.toLowerCase) val deviceLanguage = query.features.get.getOrElse(DeviceLanguageFeature, None).map(_.toLowerCase).toSet val allUserLanguages = userLanguages ++ userEngagedLanguages ++ deviceLanguage val (kept, removed) = if (!allUserLanguages.isEmpty) { candidates.partition { candidate => val inNetwork = candidate.features.getOrElse(InNetworkFeature, true) val postLanguage = candidate.features .getOrElse(TweetLanguageFromTweetypieFeature, None) .map(_.toLowerCase) postLanguage.forall { lang => inNetwork || allUserLanguages.contains(lang) || AllowedLanguageCodes.contains(lang) } } } else { (candidates, Seq.empty) } val filterResult = FilterResult( kept = kept.map(_.candidate), removed = removed.map(_.candidate) ) Stitch.value(filterResult) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OONReplyFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * This filter removes recommended replies to not followed users */ object OONReplyFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("OONReply") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val sgsFollowedUsers = query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).toSet.flatten val (removed, kept) = candidates.partition { candidate => val isValidRecommendedReply = !candidate.features.getOrElse(IsRetweetFeature, false) && candidate.features .getOrElse(InReplyToUserIdFeature, None).exists(sgsFollowedUsers.contains) val isRecommendedReply = candidate.features.getOrElse(InReplyToTweetIdFeature, None).nonEmpty && !candidate.features.getOrElse(AuthorIdFeature, None).exists(sgsFollowedUsers.contains) isRecommendedReply && !isValidRecommendedReply } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OutOfNetworkCompetitorFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CompetitorSetParam import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object OutOfNetworkCompetitorFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("OutOfNetworkCompetitor") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val competitorAuthors = query.params(CompetitorSetParam) val (removed, kept) = candidates.partition(isOutOfNetworkTweetFromCompetitor(_, competitorAuthors)) Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } def isOutOfNetworkTweetFromCompetitor( candidate: CandidateWithFeatures[TweetCandidate], competitorAuthors: Set[Long] ): Boolean = { !candidate.features.getOrElse(InNetworkFeature, true) && !candidate.features.getOrElse(IsRetweetFeature, false) && candidate.features.getOrElse(AuthorIdFeature, None).exists(competitorAuthors.contains) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/OutOfNetworkCompetitorURLFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CompetitorURLSeqParam import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object OutOfNetworkCompetitorURLFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("OutOfNetworkCompetitorURL") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val competitorUrls = query.params(CompetitorURLSeqParam).toSet val (removed, kept) = candidates.partition(hasOutOfNetworkUrlFromCompetitor(_, competitorUrls)) Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } def hasOutOfNetworkUrlFromCompetitor( candidate: CandidateWithFeatures[TweetCandidate], competitorUrls: Set[String] ): Boolean = { !candidate.features.getOrElse(InNetworkFeature, true) && !candidate.features.getOrElse(IsRetweetFeature, false) && candidate.features .getOrElse(TweetUrlsFeature, Seq.empty).toSet.intersect(competitorUrls).nonEmpty } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/QualifiedRepliesFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsInReplyToReplyOrDirectedFeature import com.twitter.home_mixer.model.HomeFeatures.IsInReplyToRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object QualifiedRepliesFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("QualifiedReplies") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val sgsFollowedUsers = query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).toSet.flatten val (removed, kept) = candidates.partition { candidate => val isRetweet = candidate.features.getOrElse(IsRetweetFeature, false) val authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L) val inReplyToUser = candidate.features.getOrElse(InReplyToUserIdFeature, None) val replyToFollowed = inReplyToUser.exists(sgsFollowedUsers.contains) val isValidReplyToUser = inReplyToUser.exists { user => user != query.getRequiredUserId && user != authorId } val inReplyToRetweetOrReplyOrDirected = candidate.features.getOrElse(IsInReplyToReplyOrDirectedFeature, false) || candidate.features.getOrElse(IsInReplyToRetweetFeature, false) inReplyToUser.nonEmpty && !replyToFollowed && sgsFollowedUsers.contains(authorId) && (isRetweet || !isValidReplyToUser || inReplyToRetweetOrReplyOrDirected) } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/RetweetSourceTweetRemovingFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.util.ReplyRetweetUtil import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * This filter removes source tweets of retweets, added via second EB call in TLR */ object RetweetSourceTweetRemovingFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("RetweetSourceTweetRemoving") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (kept, removed) = candidates.partition( _.features.getOrElse(EarlybirdFeature, None).exists(_.isSourceTweet)) match { case (sourceTweets, nonSourceTweets) => val inReplyToTweetIds: Set[Long] = nonSourceTweets .filter(ReplyRetweetUtil.isEligibleReply(_)).flatMap( _.features.getOrElse(InReplyToTweetIdFeature, None)).toSet val (keptSourceTweets, removedSourceTweets) = sourceTweets .map(_.candidate) .partition(candidate => inReplyToTweetIds.contains(candidate.id)) (nonSourceTweets.map(_.candidate) ++ keptSourceTweets, removedSourceTweets) } Stitch.value(FilterResult(kept = kept, removed = removed)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/SGSAuthorFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.socialgraph.{thriftscala => sg} import com.twitter.stitch.Stitch import com.twitter.stitch.socialgraph.SocialGraph import javax.inject.Inject import javax.inject.Singleton @Singleton case class SGSAuthorFilter @Inject() (socialGraph: SocialGraph) extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("SGSAuthor") override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val authorIds = candidates.flatMap { candidate => candidate.features.get(AuthorIdFeature) }.distinct val request = sg.IdsRequest( relationships = Seq( sg.SrcRelationship( source = query.getRequiredUserId, relationshipType = sg.RelationshipType.Blocking, hasRelationship = true, targets = Some(authorIds)), sg.SrcRelationship( source = query.getRequiredUserId, relationshipType = sg.RelationshipType.BlockedBy, hasRelationship = true, targets = Some(authorIds)), sg.SrcRelationship( source = query.getRequiredUserId, relationshipType = sg.RelationshipType.Muting, hasRelationship = true, targets = Some(authorIds)) ), pageRequest = Some(sg.PageRequest(selectAll = Some(true))), context = Some(sg.LookupContext(performUnion = Some(true))) ) socialGraph.ids(request).map { result => val ids = result.ids.toSet val (removed, kept) = candidates.partition { candidate => ids.contains(candidate.features.get(AuthorIdFeature).get) } FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate)) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/ScoredTweetsSocialContextFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelineservice.suggests.{thriftscala => st} object ScoredTweetsSocialContextFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("ScoredTweetsSocialContext") // Tweets from candidate sources which don't need generic like/follow/topic proof private val AllowedSources: Set[st.SuggestType] = Set( st.SuggestType.RankedListTweet, st.SuggestType.RecommendedTrendTweet, st.SuggestType.MediaTweet ) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val validTweetIds = candidates .filter { candidate => candidate.features.getOrElse(InNetworkFeature, true) || candidate.features.getOrElse(SuggestTypeFeature, None).exists(AllowedSources.contains) || candidate.features.getOrElse(InReplyToTweetIdFeature, None).isDefined || hasLikedBySocialContext(candidate.features) || hasFollowedBySocialContext(candidate.features) || hasTopicSocialContext(candidate.features) }.map(_.candidate.id).toSet val (kept, removed) = candidates.map(_.candidate).partition(candidate => validTweetIds.contains(candidate.id)) Stitch.value(FilterResult(kept = kept, removed = removed)) } private def hasLikedBySocialContext(candidateFeatures: FeatureMap): Boolean = candidateFeatures .getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) .exists( candidateFeatures .getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty) .toSet.contains ) private def hasFollowedBySocialContext(candidateFeatures: FeatureMap): Boolean = candidateFeatures.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty private def hasTopicSocialContext(candidateFeatures: FeatureMap): Boolean = { candidateFeatures.getOrElse(TopicIdSocialContextFeature, None).isDefined && candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/TopKFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * This filter ranks tweets by a score feature and takes top k */ case class TopKFilter(scoreFeature: Feature[_, Double], topK: Int) extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("TopK") private val DefaultScore = 0D override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val sorted = candidates .sortBy(_.features.getOrElse(scoreFeature, DefaultScore))(Ordering[Double].reverse) val (kept, removed) = sorted.splitAt(topK) Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/TopKOptionalFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * This filter ranks tweets by a score feature and takes top k */ case class TopKOptionalFilter(scoreFeature: Feature[_, Option[Double]], topK: Int) extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("TopKOptional") private val DefaultScore = 0D override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val sorted = candidates .sortBy(_.features.getOrElse(scoreFeature, None).getOrElse(DefaultScore))( Ordering[Double].reverse) val (kept, removed) = sorted.splitAt(topK) Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/UtegMinFavCountFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.product.scored_tweets.response_transformer.UtegFavListFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object UtegMinFavCountFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("UtegMinFavCount") private val UtegMinFavCount = 1 override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val (kept, removed) = candidates.partition { candidate => val nonAuthorFavList = candidate.features .getOrElse(UtegFavListFeature, Seq.empty) .filter(!candidate.features.getOrElse(AuthorIdFeature, None).contains(_)) nonAuthorFavList.size >= UtegMinFavCount } Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter/UtegTopKFilter.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.filter import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FetchParams import com.twitter.home_mixer.product.scored_tweets.response_transformer.UtegScoreFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch object UtegTopKFilter extends Filter[PipelineQuery, TweetCandidate] { override val identifier: FilterIdentifier = FilterIdentifier("UtegTopK") private val DefaultScore = 0D override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[FilterResult[TweetCandidate]] = { val topK = query.params(FetchParams.UTEGMaxTweetsToFetchParam) val sorted = candidates.sortBy { candidate => val utegScore = candidate.features.getOrElse(UtegScoreFeature, DefaultScore) val ebScore = candidate.features .getOrElse(EarlybirdScoreFeature, None) .getOrElse(DefaultScore) utegScore + ebScore * 2 }(Ordering[Double].reverse) val (kept, removed) = sorted.splitAt(topK) Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/AllowLowSignalUserGate.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.gate import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableLowSignalUserCheck import com.twitter.home_mixer.util.SignalUtil import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Continue for low signal users who also follow < 5 users * * Check gate param last to only evaluate for eligible users and avoid experimental dilution. */ object AllowLowSignalUserGate extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("AllowLowSignalUser") override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { val continue = SignalUtil.isLowSignalUser(query) && query.params(EnableLowSignalUserCheck) Stitch.value(continue) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/DenyLowSignalUserGate.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.gate import com.twitter.home_mixer.model.HomeFeatures.SignupSourceFeature import com.twitter.home_mixer.model.signup.MarchMadness import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableLowSignalUserCheck import com.twitter.home_mixer.util.SignalUtil import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Continue for all users except low signal users who also follow < 25 users * * Check gate param last to only evaluate for eligible users and avoid experimental dilution. */ object DenyLowSignalUserGate extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("DenyLowSignalUser") override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { val signupSource = query.features.flatMap(_.getOrElse(SignupSourceFeature, None)) val stop = signupSource.contains(MarchMadness) && SignalUtil.isLowSignalUser(query) && query.params(EnableLowSignalUserCheck) Stitch.value(!stop) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MatchesCountryGate.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.gate import com.twitter.home_mixer.model.HomeFeatures.SignupCountryFeature import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.Param /** * Check whether the signup country code feature or the input country code * exists within the input country codes param */ case class MatchesCountryGate(countryCodes: Param[Seq[String]]) extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("MatchesSignupCountry") /** * The main predicate that controls this gate. If this predicate returns true, the gate returns Continue. */ override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { val countryCode = query.clientContext.countryCode val signupCountryCode = query.features .flatMap(_.getOrElse(SignupCountryFeature, None)) val codeMatchesInput = Seq(countryCode, signupCountryCode).flatten.exists { code => query.params(countryCodes).map(_.toLowerCase).contains(code.toLowerCase) } Stitch.value(codeMatchesInput) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinCachedTweetsGate.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.gate import com.twitter.home_mixer.model.HomeFeatures.HasRecentFeedbackSinceCacheTtlFeature import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate.identifierSuffix import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableRecentFeedbackCheckParam import com.twitter.home_mixer.util.CachedScoredTweetsHelper import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.Param case class MinCachedTweetsGate( candidatePipelineIdentifier: CandidatePipelineIdentifier, minCachedTweetsParam: Param[Int]) extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier(candidatePipelineIdentifier + identifierSuffix) override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { val minCachedTweets = query.params(minCachedTweetsParam) val cachedScoredTweets = query.features.map(CachedScoredTweetsHelper.unseenCachedScoredTweets).getOrElse(Seq.empty) val numCachedTweets = cachedScoredTweets.count { tweet => tweet.candidatePipelineIdentifier.exists( CandidatePipelineIdentifier(_).equals(candidatePipelineIdentifier)) } val hasMinCachedTweets = numCachedTweets < minCachedTweets query.features.map(_.getOrElse(HasRecentFeedbackSinceCacheTtlFeature, false)) match { case Some(true) => if (query.params(EnableRecentFeedbackCheckParam)) Stitch.True else Stitch.value(hasMinCachedTweets) case _ => Stitch.value(hasMinCachedTweets) } } } object MinCachedTweetsGate { val identifierSuffix = "MinCachedTweets" } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/MinTimeSinceLastRequestGate.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.gate import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.LastNonPollingTimeFeature import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Gate continues if the amount of time passed since the previous request is greater * than the configured amount or if the previous request time in not available */ object MinTimeSinceLastRequestGate extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("TimeSinceLastRequest") private val MinTimeSinceLastRequest = 24.hours override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = Stitch.value { query.features.exists { features => features .getOrElse(LastNonPollingTimeFeature, None) .forall(lnpt => (query.queryTime - lnpt) > MinTimeSinceLastRequest) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate/RecentFeedbackCheckGate.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.gate import com.twitter.home_mixer.model.HomeFeatures.HasRecentFeedbackSinceCacheTtlFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableRecentFeedbackCheckParam import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** * Continue if a user has no don't like feedback within cached tweets ttl time (3 mins). * The reason is that if a user clicks don't like, tweets in the cache won't be affected * and it feels our system has slow response to user's negative feedback */ object RecentFeedbackCheckGate extends Gate[PipelineQuery] { override val identifier: GateIdentifier = GateIdentifier("RecentFeedbackCheck") override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = { query.features.map(_.getOrElse(HasRecentFeedbackSinceCacheTtlFeature, false)) match { case Some(true) => if (query.params(EnableRecentFeedbackCheckParam)) Stitch.False else Stitch.True case _ => Stitch.True } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseDomainMarshaller.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.marshaller import com.twitter.home_mixer.model.HomeFeatures.UserActionsContainsExplicitSignalsFeature import com.twitter.home_mixer.model.HomeFeatures.UserActionsSizeFeature import com.twitter.home_mixer.product.scored_tweets.model.QueryMetadata import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller import com.twitter.product_mixer.core.model.common.identifier.DomainMarshallerIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails /** * Creates a domain model of the Scored Tweets product response from the set of candidates selected */ object ScoredTweetsResponseDomainMarshaller extends DomainMarshaller[ScoredTweetsQuery, ScoredTweetsResponse] { override val identifier: DomainMarshallerIdentifier = DomainMarshallerIdentifier("ScoredTweetsResponse") override def apply( query: ScoredTweetsQuery, selections: Seq[CandidateWithDetails] ): ScoredTweetsResponse = ScoredTweetsResponse( scoredTweets = selections, queryMetadata = Some( QueryMetadata( userActionsSize = query.features.get.getOrElse(UserActionsSizeFeature, None), userActionsContainsExplicitSignals = Some(query.features.get.getOrElse(UserActionsContainsExplicitSignalsFeature, false)) )) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller/ScoredTweetsResponseTransportMarshaller.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.marshaller import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.model._ import com.twitter.home_mixer.product.scored_tweets.model.QueryMetadata import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityNameFeature import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.TopicContextFunctionalityTypeMarshaller import com.twitter.product_mixer.core.model.common.identifier.TransportMarshallerIdentifier /** * Marshall the domain model into our transport (Thrift) model. */ object ScoredTweetsResponseTransportMarshaller extends TransportMarshaller[ScoredTweetsResponse, hmt.ScoredTweetsResponse] { override val identifier: TransportMarshallerIdentifier = TransportMarshallerIdentifier("ScoredTweetsResponse") override def apply(input: ScoredTweetsResponse): hmt.ScoredTweetsResponse = { val scoredTweets = input.scoredTweets.map { tweet => mkScoredTweet(tweet.candidateIdLong, tweet.features, input.queryMetadata) } hmt.ScoredTweetsResponse(scoredTweets) } private def mkScoredTweet( tweetId: Long, features: FeatureMap, queryMetadata: Option[QueryMetadata] ): hmt.ScoredTweet = { val topicFunctionalityType = features .getOrElse(TopicContextFunctionalityTypeFeature, None) .map(TopicContextFunctionalityTypeMarshaller(_)) val predictedScores = hmt.PredictedScores( favoriteScore = features.getOrElse(PredictedFavoriteScoreFeature, None), replyScore = features.getOrElse(PredictedReplyScoreFeature, None), retweetScore = features.getOrElse(PredictedRetweetScoreFeature, None), replyEngagedByAuthorScore = features.getOrElse(PredictedReplyEngagedByAuthorScoreFeature, None), goodClickConvoDescFavoritedOrRepliedScore = features .getOrElse(PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, None), goodClickConvoDescUamGt2Score = features.getOrElse(PredictedGoodClickConvoDescUamGt2ScoreFeature, None), goodProfileClickScore = features.getOrElse(PredictedGoodProfileClickScoreFeature, None), videoQualityViewScore = features.getOrElse(PredictedVideoQualityViewScoreFeature, None), shareScore = features.getOrElse(PredictedShareScoreFeature, None), dwellScore = features.getOrElse(PredictedDwellScoreFeature, None), negativeFeedbackV2Score = features.getOrElse(PredictedNegativeFeedbackV2ScoreFeature, None) ) val phoenixPredictedScores = hmt.PredictedScores( favoriteScore = features.getOrElse(PhoenixPredictedFavoriteScoreFeature, None), replyScore = features.getOrElse(PhoenixPredictedReplyScoreFeature, None), retweetScore = features.getOrElse(PhoenixPredictedRetweetScoreFeature, None), replyEngagedByAuthorScore = None, goodClickConvoDescFavoritedOrRepliedScore = features .getOrElse(PhoenixPredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, None), goodClickConvoDescUamGt2Score = features.getOrElse(PhoenixPredictedGoodClickConvoDescUamGt2ScoreFeature, None), goodProfileClickScore = features.getOrElse(PhoenixPredictedGoodProfileClickScoreFeature, None), videoQualityViewScore = features.getOrElse(PhoenixPredictedVideoQualityViewScoreFeature, None), shareScore = features.getOrElse(PhoenixPredictedShareScoreFeature, None), dwellScore = features.getOrElse(PhoenixPredictedDwellScoreFeature, None), negativeFeedbackV2Score = features.getOrElse(PhoenixPredictedNegativeFeedbackV2ScoreFeature, None), openLinkScore = features.getOrElse(PhoenixPredictedOpenLinkScoreFeature, None), screenshotScore = features.getOrElse(PhoenixPredictedScreenshotScoreFeature, None), bookmarkScore = features.getOrElse(PhoenixPredictedBookmarkScoreFeature, None) ) val sourceSignal: Option[hmt.SourceSignal] = features.getOrElse(SourceSignalFeature, None).map { modelSignal => hmt.SourceSignal( id = modelSignal.id, signalType = modelSignal.signalType, signalEntity = modelSignal.signalEntity, authorId = modelSignal.authorId, ) } hmt.ScoredTweet( tweetId = tweetId, authorId = features.get(AuthorIdFeature).get, score = features.get(ScoreFeature), servedType = features.get(ServedTypeFeature), sourceTweetId = features.getOrElse(SourceTweetIdFeature, None), sourceUserId = features.getOrElse(SourceUserIdFeature, None), quotedTweetId = features.getOrElse(QuotedTweetIdFeature, None), quotedUserId = features.getOrElse(QuotedUserIdFeature, None), inReplyToTweetId = features.getOrElse(InReplyToTweetIdFeature, None), inReplyToUserId = features.getOrElse(InReplyToUserIdFeature, None), directedAtUserId = features.getOrElse(DirectedAtUserIdFeature, None), inNetwork = Some(features.getOrElse(InNetworkFeature, true)), sgsValidLikedByUserIds = Some(features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)), sgsValidFollowedByUserIds = Some(features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)), validLikedByUserIds = Some(features.getOrElse(ValidLikedByUserIdsFeature, Seq.empty)), topicId = features.getOrElse(TopicIdSocialContextFeature, None), topicFunctionalityType = topicFunctionalityType, ancestors = Some(features.getOrElse(AncestorsFeature, Seq.empty)), isReadFromCache = Some(features.getOrElse(IsReadFromCacheFeature, false)), exclusiveConversationAuthorId = features.getOrElse(ExclusiveConversationAuthorIdFeature, None), authorMetadata = Some( hmt.AuthorMetadata( blueVerified = features.getOrElse(AuthorIsBlueVerifiedFeature, false), goldVerified = features.getOrElse(AuthorIsGoldVerifiedFeature, false), grayVerified = features.getOrElse(AuthorIsGrayVerifiedFeature, false), legacyVerified = features.getOrElse(AuthorIsLegacyVerifiedFeature, false), creator = features.getOrElse(AuthorIsCreatorFeature, false), followers = features.getOrElse(AuthorFollowersFeature, None) )), lastScoredTimestampMs = None, candidatePipelineIdentifier = None, tweetUrls = None, perspectiveFilteredLikedByUserIds = None, predictionRequestId = features.getOrElse(PredictionRequestIdFeature, None), communityId = features.getOrElse(CommunityIdFeature, None), communityName = features.getOrElse(CommunityNameFeature, None), listId = features.getOrElse(ListIdFeature, None), listName = features.getOrElse(ListNameFeature, None), isNsfw = features.getOrElse(IsNsfw, None), visibilityReason = features.getOrElse(VisibilityReason, None), tweetLanguage = features.getOrElse(TweetLanguageFeature, None), tweetText = features.getOrElse(TweetTextFeature, None), tweetTypeMetrics = features.getOrElse(TweetTypeMetricsFeature, None), debugString = features.getOrElse(DebugStringFeature, None), hasVideo = Some(features.getOrElse(HasVideoFeature, false)), videoDurationMs = features.getOrElse(VideoDurationMsFeature, None), mediaIds = Some(features.getOrElse(TweetMediaIdsFeature, Seq.empty)), grokAnnotations = features.getOrElse(GrokAnnotationsFeature, None), predictedScores = Some(predictedScores), tweetMixerScore = features.getOrElse(TweetMixerScoreFeature, None), phoenixPredictedScores = Some(phoenixPredictedScores), sourceSignal = sourceSignal, userActionsSize = queryMetadata.flatMap(_.userActionsSize), userActionsContainsExplicitSignals = queryMetadata.flatMap(_.userActionsContainsExplicitSignals) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", ], exports = [ "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsQuery.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.model import com.twitter.home_mixer.model.request.DeviceContext import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.model.request.HasSeenTweetIds import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.request._ import com.twitter.product_mixer.core.pipeline.HasPipelineCursor import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus import com.twitter.product_mixer.core.quality_factor.QualityFactorStatus import com.twitter.timelines.configapi.Params case class ScoredTweetsQuery( override val params: Params, override val clientContext: ClientContext, override val pipelineCursor: Option[UrtOrderedCursor], override val requestedMaxResults: Option[Int], override val debugOptions: Option[DebugOptions], override val features: Option[FeatureMap], override val deviceContext: Option[DeviceContext], override val seenTweetIds: Option[Seq[Long]], override val qualityFactorStatus: Option[QualityFactorStatus], override val product: Product, videoType: Option[t.VideoType] = None, pinnedRelatedTweetIds: Option[Seq[Long]] = None, scorePinnedTweetsOnly: Option[Boolean] = None, immersiveClientMetadata: Option[t.ImmersiveClientMetadata] = None) extends PipelineQuery with HasPipelineCursor[UrtOrderedCursor] with HasDeviceContext with HasSeenTweetIds with HasQualityFactorStatus { override def withFeatureMap(features: FeatureMap): ScoredTweetsQuery = copy(features = Some(features)) override def withQualityFactorStatus( qualityFactorStatus: QualityFactorStatus ): ScoredTweetsQuery = copy(qualityFactorStatus = Some(qualityFactorStatus)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model/ScoredTweetsResponse.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.model import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.HasMarshalling import com.twitter.product_mixer.core.model.marshalling.HasLength case class ScoredTweetsResponse( scoredTweets: Seq[CandidateWithDetails], queryMetadata: Option[QueryMetadata] = None) extends HasMarshalling with HasLength { override def length: Int = scoredTweets.length } case class QueryMetadata( userActionsSize: Option[Int] = None, userActionsContainsExplicitSignals: Option[Boolean] = None) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParam.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.param import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.param.decider.DeciderKey import com.twitter.timelines.configapi.DurationConversion import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam import com.twitter.timelines.configapi.HasDurationConversion import com.twitter.timelines.configapi.decider.BooleanDeciderParam import com.twitter.timelines.configapi.decider.DeciderBoundedParam import com.twitter.util.Duration object ScoredTweetsParam { val SupportedClientFSName = "scored_tweets_supported_client" object CandidateSourceParams { object EnableCommunitiesCandidateSourceParam extends FSParam[Boolean]( name = "scored_tweets_enable_earlybird_communities_candidate_source", default = false ) object EnableInNetworkCandidateSourceParam extends FSParam[Boolean]( name = "scored_tweets_enable_in_network_candidate_source", default = true ) object EnableStaticSourceParam extends FSParam[Boolean]( name = "scored_tweets_enable_static_source", default = false ) object EnableUTEGCandidateSourceParam extends FSParam[Boolean]( name = "scored_tweets_enable_uteg_candidate_source", default = true ) object InNetworkIncludeRepliesParam extends FSParam[Boolean]( name = "scored_tweets_in_network_include_replies", default = true ) object InNetworkIncludeRetweetsParam extends FSParam[Boolean]( name = "scored_tweets_in_network_include_retweets", default = true ) object InNetworkIncludeExtendedRepliesParam extends FSParam[Boolean]( name = "scored_tweets_in_network_include_extended_replies", default = true ) } object EnableBackfillCandidatePipelineParam extends FSParam[Boolean]( name = "scored_tweets_enable_backfill_candidate_pipeline", default = true ) object EnableContentExplorationCandidatePipelineParam extends FSParam[Boolean]( name = "scored_tweets_enable_content_exploration_candidate_pipeline", default = false ) object ContentExplorationCandidateVersionParam extends FSParam[String]( name = "scored_tweets_enable_content_exploration_candidate_version", default = "v1_" ) object EnableContentExplorationScoreScribingParam extends FSParam[Boolean]( name = "scored_tweets_enable_content_exploration_score_scribing", default = false ) object EnableContentExplorationCandidateMaxCountParam extends FSParam[Boolean]( name = "scored_tweets_enable_content_exploration_candidate_max_count", default = false ) object EnableContentExplorationSimclusterColdPostsCandidateBoostingParam extends FSParam[Boolean]( name = "scored_tweets_enable_content_exploration_simcluster_cold_posts_candidate_boosting", default = false ) object ContentExplorationBoostPosParam extends FSBoundedParam[Int]( name = "scored_tweets_content_exploration_boost_pos", default = 100, min = 0, max = 1000 ) object EnableDeepRetrievalMixedCandidateBoostingParam extends FSParam[Boolean]( name = "scored_tweets_enable_deep_retrieval_mixed_candidate_boosting", default = false ) object CategoryColdStartTierOneProbabilityParam extends FSBoundedParam[Double]( name = "scored_tweets_category_cold_start_tier_one_probability", default = 0, min = 0, max = 1 ) object CategoryColdStartProbabilisticReturnParam extends FSBoundedParam[Double]( name = "scored_tweets_category_cold_start_probabilistic_return", default = 0, min = 0, max = 1 ) object ContentExplorationViewerMaxFollowersParam extends FSBoundedParam[Int]( name = "scored_tweets_content_exploration_viewer_max_followers", default = 100000, min = 0, max = 1000000000 ) object EnableContentExplorationMixedCandidateBoostingParam extends FSParam[Boolean]( name = "scored_tweets_enable_content_exploration_mixed_candidate_boosting", default = false ) object DeepRetrievalBoostPosParam extends FSBoundedParam[Int]( name = "scored_tweets_deep_retrieval_boost_pos", default = 100, min = 0, max = 1000 ) object DeepRetrievalI2iProbabilityParam extends FSBoundedParam[Double]( name = "scored_tweets_deep_retrieval_i2i_probability", default = 0, min = 0, max = 1 ) object FetchParams { object FRSMaxTweetsToFetchParam extends FSBoundedParam[Int]( name = "scored_tweets_frs_max_tweets_to_fetch", default = 100, min = 0, max = 10000 ) object InNetworkMaxTweetsToFetchParam extends FSBoundedParam[Int]( name = "scored_tweets_in_network_max_tweets_to_fetch", default = 600, min = 0, max = 10000 ) object TweetMixerMaxTweetsToFetchParam extends FSBoundedParam[Int]( name = "scored_tweets_tweet_mixer_max_tweets_to_fetch", default = 400, min = 0, max = 10000 ) object UTEGMaxTweetsToFetchParam extends FSBoundedParam[Int]( name = "scored_tweets_uteg_max_tweets_to_fetch", default = 300, min = 0, max = 10000 ) } object QualityFactor { object InNetworkMaxTweetsToScoreParam extends FSBoundedParam[Int]( name = "scored_tweets_quality_factor_earlybird_max_tweets_to_score", default = 600, min = 0, max = 10000 ) object UtegMaxTweetsToScoreParam extends FSBoundedParam[Int]( name = "scored_tweets_quality_factor_uteg_max_tweets_to_score", default = 300, min = 0, max = 10000 ) object TweetMixerMaxTweetsToScoreParam extends FSBoundedParam[Int]( name = "scored_tweets_quality_factor_tweet_mixer_max_tweets_to_score", default = 400, min = 0, max = 10000 ) object ListsMaxTweetsToScoreParam extends FSBoundedParam[Int]( name = "scored_tweets_quality_factor_lists_max_tweets_to_score", default = 100, min = 0, max = 100 ) object BackfillMaxTweetsToScoreParam extends FSBoundedParam[Int]( name = "scored_tweets_quality_factor_backfill_max_tweets_to_score", default = 200, min = 0, max = 10000 ) object CommunitiesMaxTweetsToScoreParam extends FSBoundedParam[Int]( name = "scored_tweets_quality_factor_communities_max_tweets_to_score", default = 100, min = 0, max = 10000 ) } object ServerMaxResultsParam extends FSBoundedParam[Int]( name = "scored_tweets_server_max_results", default = 50, min = 1, max = 1500 ) object DefaultRequestedMaxResultsParam extends FSBoundedParam[Int]( name = "scored_tweets_default_requested_max_results", default = 50, min = 1, max = 1500 ) object CachedScoredTweets { object TTLParam extends FSBoundedParam[Duration]( name = "scored_tweets_cached_scored_tweets_ttl_minutes", default = 3.minutes, min = 0.minute, max = 60.minutes ) with HasDurationConversion { override val durationConversion: DurationConversion = DurationConversion.FromMinutes } object MinCachedTweetsParam extends FSBoundedParam[Int]( name = "scored_tweets_cached_scored_tweets_min_cached_tweets", default = 30, min = 0, max = 1000 ) } object FeatureHydration { object EnableRealTimeEntityRealGraphFeaturesParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_real_time_entity_real_graph_features", default = false ) object EnableFollowedUserScoreBackfillFeaturesParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_followed_user_score_backfill_features", default = false ) object EnableSgsMutuallyFollowedUserFeaturesParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_sgs_mutually_followed_user_features", default = false ) object EnableTopicSocialProofFeaturesParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_topic_social_proof_features", default = false ) object EnableMediaClusterFeatureHydrationParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_media_cluster_feature", default = false ) object EnableMediaCompletionRateFeatureHydrationParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_media_completion_rate_feature", default = false ) object EnableImageClusterFeatureHydrationParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_image_cluster_feature", default = false ) object EnableClipImagesClusterIdFeatureHydrationParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_clip_images_cluster_id_feature", default = false ) object EnableMultiModalEmbeddingsFeatureHydratorParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_multi_modal_embeddings_feature_hydrator", default = false ) object EnableTweetTextV8EmbeddingFeatureParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_tweet_text_v8_embedding_feature", default = false ) object EnableUserEngagedLanguagesFeaturesParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_user_engaged_languages_features", default = false ) object EnableUserIdentifierFeaturesParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_user_identifier_features", default = false ) object EnableUserHistoryEventsFeaturesParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_user_history_events_features", default = false ) object EnableUserActionsFeatureParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_user_actions_feature", default = false ) object EnableDenseUserActionsHydrationParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_dense_user_actions_feature", default = false ) object EnableMediaClusterDecayParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_media_cluster_decay", default = false ) object EnableImageClusterDecayParam extends FSParam[Boolean]( name = "scored_tweets_feature_hydration_enable_image_cluster_decay", default = false ) object UserHistoryEventsLengthParam extends FSBoundedParam[Int]( name = "scored_tweets_feature_hydration_user_history_events_length", default = 50, min = 0, max = 1000 ) object TwhinDiversityRescoringWeightParam extends FSBoundedParam[Double]( name = "scored_tweets_feature_hydration_twhin_diversity_rescoring_weight", default = 0.0, min = -100.0, max = 100.0 ) object TwhinDiversityRescoringRatioParam extends FSBoundedParam[Double]( name = "scored_tweets_feature_hydration_twhin_diversity_rescoring_ratio", default = 0.0, min = 0.0, max = 1.0 ) object CategoryDiversityRescoringWeightParam extends FSBoundedParam[Double]( name = "scored_tweets_feature_hydration_category_diversity_rescoring_weight", default = 0.0, min = -1.0, max = 1.0 ) object CategoryDiversityKParam extends FSBoundedParam[Int]( name = "scored_tweets_feature_hydration_category_diversity_k", default = 5, min = 1, max = 100 ) } object ControlAiShowLessScaleFactorParam extends FSBoundedParam[Double]( name = "scored_tweets_control_ai_show_less_scale_factor", default = 0.05, min = 0.0, max = 1.0 ) object ControlAiShowMoreScaleFactorParam extends FSBoundedParam[Double]( name = "scored_tweets_control_ai_show_more_scale_factor", default = 20.0, min = 0.0, max = 1000.0 ) object ControlAiEmbeddingSimilarityThresholdParam extends FSBoundedParam[Double]( name = "scored_tweets_control_ai_embedding_similarity_threshold", default = 0.67, min = 0.0, max = 1.0 ) object CreatorInNetworkMultiplierParam extends FSBoundedParam[Double]( name = "scored_tweets_creator_in_network_multiplier", default = 1.0, min = 0.0, max = 100.0 ) object CreatorOutOfNetworkMultiplierParam extends FSBoundedParam[Double]( name = "scored_tweets_creator_out_of_network_multiplier", default = 1.0, min = 0.0, max = 100.0 ) object OutOfNetworkScaleFactorParam extends FSBoundedParam[Double]( name = "scored_tweets_out_of_network_scale_factor", default = 0.75, min = 0.0, max = 100.0 ) object ReplyScaleFactorParam extends FSBoundedParam[Double]( name = "scored_tweets_reply_scale_factor", default = 0.75, min = 0.0, max = 100.0 ) object EnableMediaDedupingParam extends FSParam[Boolean]( name = "scored_tweets_enable_media_deduping", default = false ) object EnableMediaClusterDedupingParam extends FSParam[Boolean]( name = "scored_tweets_enable_media_cluster_deduping", default = false ) object EnableClipImageClusterDedupingParam extends FSParam[Boolean]( name = "scored_tweets_enable_clip_image_cluster_deduping", default = false ) object EnableScribeScoredCandidatesParam extends FSParam[Boolean]( name = "scored_tweets_enable_scribing", default = false ) object EnableCacheRetrievalSignalParam extends FSParam[Boolean]( name = "scored_tweets_cache_retrieval_signal", default = false ) object EnableCacheRequestInfoParam extends FSParam[Boolean]( name = "scored_tweets_cache_request_info_signal", default = false ) object EnableScoredPhoenixCandidatesKafkaSideEffectParam extends FSParam[Boolean]( name = "scored_tweets_scored_phoenix_candidates_kafka_side_effect", default = false ) object LiveContentScaleFactorParam extends DeciderBoundedParam[Double]( DeciderKey.LiveSpacesFactor, default = 1.0, min = 0.1, max = 10000.0 ) object EarlybirdTensorflowModel { object InNetworkParam extends FSParam[String]( name = "scored_tweets_in_network_earlybird_tensorflow_model", default = "timelines_recap_replica" ) object FrsParam extends FSParam[String]( name = "scored_tweets_frs_earlybird_tensorflow_model", default = "timelines_rectweet_replica" ) object UtegParam extends FSParam[String]( name = "scored_tweets_uteg_earlybird_tensorflow_model", default = "timelines_rectweet_replica" ) } object MtlNormalization { object EnableMtlNormalizationParam extends FSParam[Boolean]( name = "scored_tweets_enable_mtl_normalization", default = true ) object AlphaParam extends DeciderBoundedParam[Double]( decider = DeciderKey.MtlNormalizationAlpha, default = 100.0, min = 0.0, max = 100.0 ) object BetaParam extends FSBoundedParam[Long]( name = "scored_tweets_mtl_normalization_beta", default = 100000000L, min = 0L, max = 1000000000L ) object GammaParam extends FSBoundedParam[Long]( name = "scored_tweets_mtl_normalization_gamma", default = 5000000L, min = 1L, max = 100000000L ) } object EarlybirdMaxResultsPerPartitionParam extends FSBoundedParam[Int]( name = "scored_tweets_earlybird_max_results_per_partition", default = 300, min = 0, max = 1000 ) object TweetMixerRankingModeForStatsRecallAtKParam extends FSParam[String]( name = "scored_tweets_tweet_mixer_ranking_mode_for_stats_recall_at_k", default = "Interleave" ) object EnablePublishCommonFeaturesKafkaDeciderParam extends BooleanDeciderParam(decider = DeciderKey.EnablePublishCommonFeaturesKafka) object AuthorDiversityDecayFactor extends FSBoundedParam[Double]( name = "scored_tweets_author_diversity_decay_factor", default = 0.5, min = 0.0, max = 1.0, ) object AuthorDiversityOutNetworkDecayFactor extends FSBoundedParam[Double]( name = "scored_tweets_author_diversity_out_network_decay_factor", default = 0.5, min = 0.0, max = 1.0, ) object AuthorDiversityInNetworkDecayFactor extends FSBoundedParam[Double]( name = "scored_tweets_author_diversity_in_network_decay_factor", default = 0.5, min = 0.0, max = 1.0, ) object AuthorDiversityFloor extends FSBoundedParam[Double]( name = "scored_tweets_author_diversity_floor", default = 0.25, min = 0.0, max = 1.0, ) object AuthorDiversityOutNetworkFloor extends FSBoundedParam[Double]( name = "scored_tweets_author_diversity_out_network_floor", default = 0.25, min = 0.0, max = 1.0, ) object AuthorDiversityInNetworkFloor extends FSBoundedParam[Double]( name = "scored_tweets_author_diversity_in_network_floor", default = 0.25, min = 0.0, max = 1.0, ) object SmallFollowGraphAuthorDiversityDecayFactor extends FSBoundedParam[Double]( name = "scored_tweets_small_follow_graph_author_diversity_decay_factor", default = 0.5, min = 0.0, max = 1.0, ) object SmallFollowGraphAuthorDiversityFloor extends FSBoundedParam[Double]( name = "scored_tweets_small_follow_graph_author_diversity_floor", default = 0.25, min = 0.0, max = 1.0, ) object EnableDeepRetrievalMaxCountParam extends FSParam[Boolean]( name = "scored_tweets_enable_deep_retrieval_max_count", default = false ) object DeepRetrievalMaxCountParam extends FSBoundedParam[Int]( name = "scored_tweets_deep_retrieval_max_count", default = 1, min = 0, max = 1000 ) object EnableEvergreenDeepRetrievalMaxCountParam extends FSParam[Boolean]( name = "scored_tweets_enable_evergreen_deep_retrieval_max_count", default = false ) object EvergreenDeepRetrievalMaxCountParam extends FSBoundedParam[Int]( name = "scored_tweets_evergreen_deep_retrieval_max_count", default = 1, min = 0, max = 1000 ) object EnableEvergreenDeepRetrievalCrossBorderMaxCountParam extends FSParam[Boolean]( name = "scored_tweets_enable_evergreen_deep_retrieval_cross_border_max_count", default = false ) object EvergreenDeepRetrievalCrossBorderMaxCountParam extends FSBoundedParam[Int]( name = "scored_tweets_evergreen_deep_retrieval_cross_border_max_count", default = 1, min = 0, max = 1000 ) object EnableControlAiParam extends FSParam[Boolean]( name = "scored_tweets_enable_control_ai", default = false ) object EnableHeartbeatOptimizerWeightsParam extends FSParam[Boolean]( name = "scored_tweets_enable_heartbeat_optimizer_weights", default = false ) object HeartbeatOptimizerParamsMHPkey extends FSParam[String]( name = "scored_tweets_heartbeat_optimizer_params_mh_pkey", default = "0" ) object EnableHeuristicScoringPipeline extends FSParam[Boolean]( name = "scored_tweets_enable_heuristic_scoring_pipeline", default = true ) object EnablePhoenixScoreParam extends FSParam[Boolean]( name = "scored_tweets_enable_phoenix_score", default = false ) object EnablePhoenixRescoreParam extends FSParam[Boolean]( name = "scored_tweets_enable_phoenix_rescore", default = false ) object EnableColdStartFilterParam extends FSParam[Boolean]( name = "scored_tweets_enable_cold_start_filter", default = false ) object EnableImpressionBasedAuthorDecay extends FSParam[Boolean]( name = "scored_tweets_enable_impression_based_author_decay", default = false ) object EnableCandidateSourceDiversityDecay extends FSParam[Boolean]( name = "scored_tweets_enable_candidate_source_diversity_decay", default = false ) object CandidateSourceDiversityDecayFactor extends FSBoundedParam[Double]( name = "scored_tweets_candidate_source_diversity_decay_factor", default = 0.9, min = 0.0, max = 1.0, ) object CandidateSourceDiversityFloor extends FSBoundedParam[Double]( name = "scored_tweets_candidate_source_diversity_floor", default = 0.8, min = 0.0, max = 1.0, ) object EnableHomeMixerFeaturesService extends FSParam[Boolean]( name = "scored_tweets_enable_home_mixer_features_service", default = false ) object GrokSlopScoreDecayValueParam extends FSBoundedParam[Double]( name = "scored_tweets_grok_slop_score_decay_value", default = 1.0, min = 0.0, max = 1.0 ) object MultiModalEmbeddingRescorerGammaParam extends FSBoundedParam[Double]( name = "scored_tweets_multi_modal_embedding_rescorer_gamma", default = 0.0, min = 0.0, max = 1.0 ) object MultiModalEmbeddingRescorerMinScoreParam extends FSBoundedParam[Double]( name = "scored_tweets_multi_modal_embedding_rescorer_min_score", default = 1.0, min = 0.0, max = 1.0 ) object EnableContentFeatureFromTesService extends FSParam[Boolean]( name = "scored_tweets_enable_home_mixer_feature_tweet_entity_service", default = false ) object EnableLowSignalUserCheck extends FSParam[Boolean]( name = "scored_tweets_enable_low_signal_user_check", default = false ) object LowSignalUserMaxSignalCount extends FSBoundedParam[Int]( name = "scored_tweets_low_signal_user_max_signal_count", default = 10, min = 0, max = 100000 ) object MultiModalEmbeddingRescorerNumCandidatesParam extends FSBoundedParam[Int]( name = "scored_tweets_multi_modal_embedding_rescorer_num_candidates", default = 70, min = 1, max = 500 ) object EnableScoredCandidateFeatureKeysKafkaPublishingParam extends FSParam[Boolean]( name = "scored_tweets_enable_scored_candidate_feature_keys_kafka_publishing", default = false ) object EnableEarlybirdCommunitiesQueryLinearRankingParam extends FSParam[Boolean]( name = "scored_tweets_enable_earlybird_communities_query_linear_ranking", default = false ) object EarlyBirdCommunitiesMaxSearchResultsParam extends FSBoundedParam[Int]( name = "scored_tweets_earlybird_communities_max_search_results", default = 100, min = 0, max = 1000 ) object EnableRecentFeedbackCheckParam extends FSParam[Boolean]( name = "scored_tweets_enable_recent_feedback_check", default = false ) object ScribedScoredCandidateNumParam extends FSBoundedParam[Int]( name = "scored_tweets_scribed_scored_candidate_num", default = 2, min = 0, max = 2000 ) object EnableRecentEngagementCacheRefreshParam extends FSParam[Boolean]( name = "scored_tweets_enable_recent_engagement_cache_refresh", default = false ) object EnableLanguageFilter extends FSParam[Boolean]( name = "scored_tweets_enable_language_filter", default = false ) object EnableGrokAutoTranslateLanguageFilter extends FSParam[Boolean]( name = "scored_tweets_enable_grok_auto_translate_language_filter", default = false ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param/ScoredTweetsParamConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.param import com.twitter.home_mixer.param.decider.DeciderKey import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam._ import com.twitter.product_mixer.core.product.ProductParamConfig import com.twitter.servo.decider.DeciderKeyName import com.twitter.timelines.configapi.FSName import com.twitter.timelines.configapi.Param import javax.inject.Inject import javax.inject.Singleton @Singleton class ScoredTweetsParamConfig @Inject() () extends ProductParamConfig { override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableScoredTweetsProduct override val supportedClientFSName: String = SupportedClientFSName override val booleanDeciderOverrides = Seq(EnablePublishCommonFeaturesKafkaDeciderParam) override val boundedDoubleDeciderOverrides = Seq( LiveContentScaleFactorParam, MtlNormalization.AlphaParam ) override val booleanFSOverrides = Seq( CandidateSourceParams.EnableCommunitiesCandidateSourceParam, CandidateSourceParams.EnableInNetworkCandidateSourceParam, CandidateSourceParams.InNetworkIncludeRepliesParam, CandidateSourceParams.InNetworkIncludeRetweetsParam, CandidateSourceParams.InNetworkIncludeExtendedRepliesParam, CandidateSourceParams.EnableUTEGCandidateSourceParam, CandidateSourceParams.EnableStaticSourceParam, EnableBackfillCandidatePipelineParam, EnableContentExplorationCandidatePipelineParam, EnableContentExplorationSimclusterColdPostsCandidateBoostingParam, EnableContentExplorationCandidateMaxCountParam, EnableContentExplorationScoreScribingParam, EnableContentExplorationMixedCandidateBoostingParam, EnableDeepRetrievalMixedCandidateBoostingParam, EnableScribeScoredCandidatesParam, EnableCacheRetrievalSignalParam, EnableCacheRequestInfoParam, EnableScoredPhoenixCandidatesKafkaSideEffectParam, FeatureHydration.EnableFollowedUserScoreBackfillFeaturesParam, FeatureHydration.EnableRealTimeEntityRealGraphFeaturesParam, FeatureHydration.EnableTopicSocialProofFeaturesParam, FeatureHydration.EnableTweetTextV8EmbeddingFeatureParam, FeatureHydration.EnableMediaClusterFeatureHydrationParam, FeatureHydration.EnableMediaCompletionRateFeatureHydrationParam, FeatureHydration.EnableClipImagesClusterIdFeatureHydrationParam, FeatureHydration.EnableMultiModalEmbeddingsFeatureHydratorParam, FeatureHydration.EnableUserEngagedLanguagesFeaturesParam, FeatureHydration.EnableUserIdentifierFeaturesParam, FeatureHydration.EnableUserHistoryEventsFeaturesParam, FeatureHydration.EnableUserActionsFeatureParam, FeatureHydration.EnableDenseUserActionsHydrationParam, FeatureHydration.EnableMediaClusterDecayParam, FeatureHydration.EnableImageClusterDecayParam, EnableControlAiParam, EnableHeartbeatOptimizerWeightsParam, EnableHeuristicScoringPipeline, EnablePhoenixScoreParam, EnablePhoenixRescoreParam, EnableColdStartFilterParam, EnableImpressionBasedAuthorDecay, EnableCandidateSourceDiversityDecay, EnableHomeMixerFeaturesService, EnableContentFeatureFromTesService, EnableLowSignalUserCheck, EnableDeepRetrievalMaxCountParam, EnableEvergreenDeepRetrievalMaxCountParam, EnableEvergreenDeepRetrievalCrossBorderMaxCountParam, EnableScoredCandidateFeatureKeysKafkaPublishingParam, EnableEarlybirdCommunitiesQueryLinearRankingParam, EnableRecentFeedbackCheckParam, EnableMediaDedupingParam, EnableMediaClusterDedupingParam, EnableClipImageClusterDedupingParam, MtlNormalization.EnableMtlNormalizationParam, EnableRecentEngagementCacheRefreshParam, EnableLanguageFilter, EnableGrokAutoTranslateLanguageFilter ) override val boundedIntFSOverrides = Seq( CachedScoredTweets.MinCachedTweetsParam, EarlybirdMaxResultsPerPartitionParam, ServerMaxResultsParam, DefaultRequestedMaxResultsParam, ContentExplorationBoostPosParam, ContentExplorationViewerMaxFollowersParam, DeepRetrievalBoostPosParam, FeatureHydration.UserHistoryEventsLengthParam, FetchParams.FRSMaxTweetsToFetchParam, FetchParams.InNetworkMaxTweetsToFetchParam, FetchParams.TweetMixerMaxTweetsToFetchParam, FetchParams.UTEGMaxTweetsToFetchParam, QualityFactor.BackfillMaxTweetsToScoreParam, QualityFactor.TweetMixerMaxTweetsToScoreParam, QualityFactor.InNetworkMaxTweetsToScoreParam, QualityFactor.ListsMaxTweetsToScoreParam, QualityFactor.UtegMaxTweetsToScoreParam, QualityFactor.CommunitiesMaxTweetsToScoreParam, DeepRetrievalMaxCountParam, EvergreenDeepRetrievalMaxCountParam, EvergreenDeepRetrievalCrossBorderMaxCountParam, ScribedScoredCandidateNumParam, LowSignalUserMaxSignalCount, EarlyBirdCommunitiesMaxSearchResultsParam, FeatureHydration.CategoryDiversityKParam, MultiModalEmbeddingRescorerNumCandidatesParam ) override val boundedLongFSOverrides = Seq( MtlNormalization.BetaParam, MtlNormalization.GammaParam, ) override val boundedDurationFSOverrides = Seq( CachedScoredTweets.TTLParam, ) override val stringFSOverrides = Seq( ContentExplorationCandidateVersionParam, EarlybirdTensorflowModel.InNetworkParam, EarlybirdTensorflowModel.FrsParam, EarlybirdTensorflowModel.UtegParam, HeartbeatOptimizerParamsMHPkey, TweetMixerRankingModeForStatsRecallAtKParam, ) override val boundedDoubleFSOverrides = Seq( CategoryColdStartTierOneProbabilityParam, CategoryColdStartProbabilisticReturnParam, ControlAiShowLessScaleFactorParam, ControlAiShowMoreScaleFactorParam, ControlAiEmbeddingSimilarityThresholdParam, CreatorInNetworkMultiplierParam, CreatorOutOfNetworkMultiplierParam, DeepRetrievalI2iProbabilityParam, OutOfNetworkScaleFactorParam, ReplyScaleFactorParam, AuthorDiversityDecayFactor, AuthorDiversityOutNetworkDecayFactor, AuthorDiversityInNetworkDecayFactor, SmallFollowGraphAuthorDiversityDecayFactor, AuthorDiversityFloor, AuthorDiversityOutNetworkFloor, AuthorDiversityInNetworkFloor, CandidateSourceDiversityDecayFactor, CandidateSourceDiversityFloor, SmallFollowGraphAuthorDiversityFloor, FeatureHydration.TwhinDiversityRescoringWeightParam, FeatureHydration.TwhinDiversityRescoringRatioParam, FeatureHydration.CategoryDiversityRescoringWeightParam, GrokSlopScoreDecayValueParam, MultiModalEmbeddingRescorerGammaParam, MultiModalEmbeddingRescorerMinScoreParam ) override def longSetFSOverrides: Seq[Param[Set[Long]] with FSName] = Seq.empty } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", "src/thrift/com/twitter/timelineranker:thrift-scala", "timelineranker/common/src/main/scala/com/twitter/timelineranker/model", "timelines/src/main/scala/com/twitter/timelines/common/model", "timelines/src/main/scala/com/twitter/timelines/earlybird/common/options", "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", "timelines/src/main/scala/com/twitter/timelines/model/candidate", "timelineservice/common:model", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/ContentExplorationQueryTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.query_transformer import com.twitter.home_mixer.functional_component.feature_hydrator.UserSubLevelCategoriesFeature import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ContentExplorationCandidateVersionParam import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer case class ContentExplorationQueryRequest( userCategories: Seq[(String, Double)], version: String) object ContentExplorationQueryTransformer extends CandidatePipelineQueryTransformer[ ScoredTweetsQuery, ContentExplorationQueryRequest ] { override def transform(query: ScoredTweetsQuery): ContentExplorationQueryRequest = { ContentExplorationQueryRequest( userCategories = query.features.get .getOrElse(UserSubLevelCategoriesFeature, Seq.empty[(Long, Double)]) .map { case (id, score) => (id.toString, score) }, version = query.params(ContentExplorationCandidateVersionParam) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerFrsQueryTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.query_transformer import com.twitter.conversions.DurationOps._ import com.twitter.core_workflows.user_model.{thriftscala => um} import com.twitter.home_mixer.functional_component.feature_hydrator.FrsSeedUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FetchParams import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerFrsQueryTransformer._ import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus import com.twitter.timelineranker.{thriftscala => t} import com.twitter.timelines.common.model.TweetKindOption import com.twitter.timelines.model.candidate.CandidateTweetSourceId object TimelineRankerFrsQueryTransformer { private val DefaultSinceDuration = 24.hours private val ExpandedSinceDuration = 48.hours private val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOption(includeOriginalTweetsAndQuotes = true) private val UserStatesForExtendedSinceDuration: Set[um.UserState] = Set( um.UserState.Light, um.UserState.MediumNonTweeter, um.UserState.MediumTweeter, um.UserState.NearZero, um.UserState.New, um.UserState.VeryLight ) } case class TimelineRankerFrsQueryTransformer[ Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext ]( override val candidatePipelineIdentifier: CandidatePipelineIdentifier) extends CandidatePipelineQueryTransformer[Query, t.RecapQuery] with TimelineRankerQueryTransformer[Query] { override val candidateTweetSourceId = CandidateTweetSourceId.FrsTweet override val options = tweetKindOptions override def maxTweetsToFetch(query: Query): Int = query.params(FetchParams.FRSMaxTweetsToFetchParam) override def getTensorflowModel(query: Query): Option[String] = { Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.FrsParam)) } override def seedAuthorIds(query: Query): Option[Seq[Long]] = { query.features.flatMap(_.getOrElse(FrsSeedUserIdsFeature, None)) } override def transform(input: Query): t.RecapQuery = { val userState = input.features.get.getOrElse(UserStateFeature, None) val sinceDuration = if (userState.exists(UserStatesForExtendedSinceDuration.contains)) ExpandedSinceDuration else DefaultSinceDuration buildTimelineRankerQuery(input, sinceDuration).toThriftRecapQuery } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerInNetworkQueryTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.query_transformer import com.twitter.conversions.DurationOps._ import com.twitter.core_workflows.user_model.{thriftscala => um} import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FetchParams import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerInNetworkQueryTransformer._ import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus import com.twitter.timelineranker.{thriftscala => t} import com.twitter.timelines.common.model.TweetKindOption import com.twitter.timelines.model.candidate.CandidateTweetSourceId object TimelineRankerInNetworkQueryTransformer { private val DefaultSinceDuration = 24.hours private val ExpandedSinceDuration = 48.hours private val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOption( includeReplies = true, includeRetweets = true, includeOriginalTweetsAndQuotes = true, includeExtendedReplies = true ) private val UserStatesForExtendedSinceDuration: Set[um.UserState] = Set( um.UserState.Light, um.UserState.MediumNonTweeter, um.UserState.MediumTweeter, um.UserState.NearZero, um.UserState.New, um.UserState.VeryLight ) } case class TimelineRankerInNetworkQueryTransformer[ Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext ]( override val candidatePipelineIdentifier: CandidatePipelineIdentifier) extends CandidatePipelineQueryTransformer[Query, t.RecapQuery] with TimelineRankerQueryTransformer[Query] { override val candidateTweetSourceId = CandidateTweetSourceId.RecycledTweet override val options = tweetKindOptions override def maxTweetsToFetch(query: Query): Int = query.params(FetchParams.InNetworkMaxTweetsToFetchParam) override def getTensorflowModel(query: Query): Option[String] = { Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.InNetworkParam)) } override def transform(input: Query): t.RecapQuery = { val userState = input.features.get.getOrElse(UserStateFeature, None) val sinceDuration = if (userState.exists(UserStatesForExtendedSinceDuration.contains)) ExpandedSinceDuration else DefaultSinceDuration buildTimelineRankerQuery(input, sinceDuration).toThriftRecapQuery } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerQueryTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.query_transformer import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EarlybirdMaxResultsPerPartitionParam import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerQueryTransformer._ import com.twitter.home_mixer.util.CachedScoredTweetsHelper import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus import com.twitter.timelineranker.{model => tlr} import com.twitter.timelines.common.model.TweetKindOption import com.twitter.timelines.earlybird.common.options.EarlybirdOptions import com.twitter.timelines.earlybird.common.options.EarlybirdScoringModelConfig import com.twitter.timelines.earlybird.common.utils.SearchOperator import com.twitter.timelines.model.UserId import com.twitter.timelines.model.candidate.CandidateTweetSourceId import com.twitter.timelines.util.SnowflakeSortIndexHelper import com.twitter.util.Duration import com.twitter.util.Time object TimelineRankerQueryTransformer { /** * Specifies the maximum number of excluded tweet ids to include in the search index query. * Earlybird's named multi term disjunction map feature supports up to 1500 tweet ids. */ private val EarlybirdMaxExcludedTweets = 1500 /** * Maximum number of query hits each earlybird shard is allowed to accumulate before * early-terminating the query and reducing the hits to MaxNumEarlybirdResults. */ private val EarlybirdMaxHits = 1000 } trait TimelineRankerQueryTransformer[ Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext] { def maxTweetsToFetch(query: Query): Int def options: TweetKindOption.ValueSet = TweetKindOption.Default def candidateTweetSourceId: CandidateTweetSourceId.Value def utegLikedByTweetsOptions(query: Query): Option[tlr.UtegLikedByTweetsOptions] = None def seedAuthorIds(query: Query): Option[Seq[Long]] = None def candidatePipelineIdentifier: CandidatePipelineIdentifier def earlybirdModels: Seq[EarlybirdScoringModelConfig] = EarlybirdRequestUtil.EarlybirdScoringModels.UnifiedEngagementProd def getTensorflowModel(query: Query): Option[String] = None def buildTimelineRankerQuery(query: Query, sinceDuration: Duration): tlr.RecapQuery = { val sinceTime: Time = sinceDuration.ago val untilTime: Time = Time.now val fromTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(sinceTime) val toTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(untilTime) val range = tlr.TweetIdRange(Some(fromTweetIdExclusive), Some(toTweetIdExclusive)) val excludedTweetIds = query.features.map { featureMap => CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweetsInRange( featureMap, candidatePipelineIdentifier, EarlybirdMaxExcludedTweets, sinceTime, untilTime) } val authorScoreMap = query.features .map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[UserId, Double])) .getOrElse(Map.empty) val deviceContext = query.deviceContext.map(_.toTimelineServiceDeviceContext(query.clientContext)) val tensorflowModel = getTensorflowModel(query) val earlyBirdOptions = EarlybirdOptions( maxNumHitsPerShard = EarlybirdMaxHits, maxNumResultsPerShard = query.params(EarlybirdMaxResultsPerPartitionParam), models = earlybirdModels, authorScoreMap = authorScoreMap, skipVeryRecentTweets = true, tensorflowModel = tensorflowModel ) tlr.RecapQuery( userId = query.getRequiredUserId, maxCount = Some(maxTweetsToFetch(query)), range = Some(range), options = options, searchOperator = SearchOperator.Exclude, earlybirdOptions = Some(earlyBirdOptions), deviceContext = deviceContext, authorIds = seedAuthorIds(query), excludedTweetIds = excludedTweetIds, utegLikedByTweetsOptions = utegLikedByTweetsOptions(query), searchClientSubId = None, candidateTweetSourceId = Some(candidateTweetSourceId), hydratesContentFeatures = Some(false) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/TimelineRankerUtegQueryTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.query_transformer import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FetchParams import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerUtegQueryTransformer._ import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus import com.twitter.timelineranker.{model => tlr} import com.twitter.timelineranker.{thriftscala => t} import com.twitter.timelines.common.model.TweetKindOption import com.twitter.timelines.earlybird.common.options.EarlybirdScoringModelConfig import com.twitter.timelines.model.UserId import com.twitter.timelines.model.candidate.CandidateTweetSourceId object TimelineRankerUtegQueryTransformer { private val SinceDuration = 24.hours private val MaxUtegCandidates = 800 private val tweetKindOptions = TweetKindOption(includeOriginalTweetsAndQuotes = true, includeReplies = true) def utegEarlybirdModels: Seq[EarlybirdScoringModelConfig] = EarlybirdRequestUtil.EarlybirdScoringModels.UnifiedEngagementRectweet } case class TimelineRankerUtegQueryTransformer[ Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext ]( override val candidatePipelineIdentifier: CandidatePipelineIdentifier) extends CandidatePipelineQueryTransformer[Query, t.UtegLikedByTweetsQuery] with TimelineRankerQueryTransformer[Query] { override val candidateTweetSourceId = CandidateTweetSourceId.RecommendedTweet override val options = tweetKindOptions override val earlybirdModels = utegEarlybirdModels override def maxTweetsToFetch(query: Query): Int = query.params(FetchParams.UTEGMaxTweetsToFetchParam) override def getTensorflowModel(query: Query): Option[String] = { Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.UtegParam)) } override def utegLikedByTweetsOptions(input: Query): Option[tlr.UtegLikedByTweetsOptions] = Some( tlr.UtegLikedByTweetsOptions( utegCount = MaxUtegCandidates, isInNetwork = false, weightedFollowings = input.features .map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[UserId, Double])) .getOrElse(Map.empty) ) ) override def transform(input: Query): t.UtegLikedByTweetsQuery = buildTimelineRankerQuery(input, SinceDuration).toThriftUtegLikedByTweetsQuery } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/UtegQueryTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.query_transformer import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.product.scored_tweets.query_transformer.UtegQueryTransformer._ import com.twitter.home_mixer.util.CachedScoredTweetsHelper import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus import com.twitter.recos.recos_common.thriftscala.SocialProofType import com.twitter.recos.user_tweet_entity_graph.thriftscala.RecommendationType import com.twitter.recos.user_tweet_entity_graph.thriftscala.TweetEntityDisplayLocation import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} import com.twitter.util.Time object UtegQueryTransformer { private val MaxUserSocialProofSize = 10 private val MaxTweetSocialProofSize = 10 private val MinUserSocialProofSize = 1 private val DefaultSinceDuration = 24.hours private val MaxTweetsToFetch = 800 private val MaxExcludedTweets = 1500 } case class UtegQueryTransformer[ Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext ]( candidatePipelineIdentifier: CandidatePipelineIdentifier) extends CandidatePipelineQueryTransformer[Query, uteg.RecommendTweetEntityRequest] { override def transform(query: Query): uteg.RecommendTweetEntityRequest = { val weightedFollowings = query.features .map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double])) .getOrElse(Map.empty) val sinceTime: Time = DefaultSinceDuration.ago val untilTime: Time = Time.now val excludedTweetIds = query.features.map { featureMap => CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweetsInRange( featureMap, candidatePipelineIdentifier, MaxExcludedTweets, sinceTime, untilTime) } uteg.RecommendTweetEntityRequest( requesterId = query.getRequiredUserId, displayLocation = TweetEntityDisplayLocation.HomeTimeline, recommendationTypes = Seq(RecommendationType.Tweet), seedsWithWeights = weightedFollowings, maxResultsByType = Some(Map(RecommendationType.Tweet -> MaxTweetsToFetch)), maxTweetAgeInMillis = Some(sinceTime.untilNow.inMillis), excludedTweetIds = excludedTweetIds, maxUserSocialProofSize = Some(MaxUserSocialProofSize), maxTweetSocialProofSize = Some(MaxTweetSocialProofSize), minUserSocialProofSizes = Some(Map(RecommendationType.Tweet -> MinUserSocialProofSize)), socialProofTypes = Some(Seq(SocialProofType.Favorite)), tweetAuthors = None, maxEngagementAgeInMillis = None, tweetTypes = None ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", "src/java/com/twitter/search/queryparser/query:core-query-nodes", "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", "src/thrift/com/twitter/search:earlybird-scala", "timelines:util", "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", "timelines/src/main/scala/com/twitter/timelines/common/model", "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", "timelines/src/main/scala/com/twitter/timelines/model/types", "timelines/src/main/scala/com/twitter/timelines/util/stats", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/CommunitiesEarlybirdQueryTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird import com.twitter.conversions.DurationOps._ import com.twitter.finagle.thrift.ClientId import com.twitter.finagle.tracing.Trace import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EarlyBirdCommunitiesMaxSearchResultsParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableEarlybirdCommunitiesQueryLinearRankingParam import com.twitter.home_mixer.util.earlybird.RelevanceSearchUtil import com.twitter.product_mixer.component_library.feature_hydrator.query.communities.CommunityMembershipsFeature import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.search.common.query.thriftjava.thriftscala.CollectorParams import com.twitter.search.common.query.thriftjava.thriftscala.CollectorTerminationParams import com.twitter.search.common.ranking.{thriftscala => scr} import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.search.earlybird.{thriftscala => t} import com.twitter.search.queryparser.query.Conjunction import com.twitter.search.queryparser.query.Disjunction import com.twitter.search.queryparser.query.search.SearchOperator import com.twitter.search.queryparser.query.search.SearchOperatorConstants import com.twitter.search.queryparser.query.{Query => SearchQuery} import javax.inject.Inject import javax.inject.Singleton import scala.collection.JavaConverters._ @Singleton class CommunitiesEarlybirdQueryTransformer @Inject() (clientId: ClientId) extends CandidatePipelineQueryTransformer[PipelineQuery, eb.EarlybirdRequest] { private val SinceDuration = 48.hours private val DefaultSearchProcessingTimeout = 200.milliseconds private val TensorflowModel = "timelines_unified_prod" override def transform(query: PipelineQuery): t.EarlybirdRequest = { val maxSearchResults = query.params(EarlyBirdCommunitiesMaxSearchResultsParam) val communityIds = query.features.map(_.getOrElse(CommunityMembershipsFeature, Seq.empty)).toSeq.flatten.toSet val entityIdsQuery = createEntityIdsQuery(communityIds) val nullcastQuery = new SearchOperator(SearchOperator.Type.INCLUDE, SearchOperatorConstants.NULLCAST) val excludeRepliesQuery = new SearchOperator(SearchOperator.Type.EXCLUDE, SearchOperatorConstants.REPLIES) val sinceTimeQuery = new SearchOperator(SearchOperator.Type.SINCE_TIME, SinceDuration.ago.inSeconds.toString) val searchQuery = new Conjunction(entityIdsQuery, excludeRepliesQuery, nullcastQuery, sinceTimeQuery) val metadataOptions = t.ThriftSearchResultMetadataOptions( getResultLocation = false, getLuceneScore = false, getInReplyToStatusId = true, getReferencedTweetAuthorId = true, getMediaBits = true, getAllFeatures = true, returnSearchResultFeatures = true, getFromUserId = true ) val ebRankingParams = Some( scr.ThriftRankingParams( `type` = Some(scr.ThriftScoringFunctionType.TensorflowBased), selectedTensorflowModel = Some(TensorflowModel), minScore = -1.0e100, applyBoosts = false, ) ) val linearRelevanceOptions = t.ThriftSearchRelevanceOptions( rankingParams = Some( scr.ThriftRankingParams( `type` = Some(scr.ThriftScoringFunctionType.Linear), applyBoosts = false, favCountParams = Some(scr.ThriftLinearFeatureRankingParams(weight = 1000.0)), replyCountParams = Some(scr.ThriftLinearFeatureRankingParams(weight = 10000.0)), quotedCountParams = Some(scr.ThriftLinearFeatureRankingParams(weight = 1000.0)) ) ), returnAllResults = Some(false) ) val relOptions = RelevanceSearchUtil.RelevanceOptions.copy( rankingParams = ebRankingParams, returnAllResults = Some(false) ) val relevanceOptions = if (query.params(EnableEarlybirdCommunitiesQueryLinearRankingParam)) linearRelevanceOptions else relOptions val collectorParams = CollectorParams( numResultsToReturn = maxSearchResults, terminationParams = Some( CollectorTerminationParams( timeoutMs = DefaultSearchProcessingTimeout.inMilliseconds.toInt ) ) ) t.EarlybirdRequest( searchQuery = t.ThriftSearchQuery( serializedQuery = Some(searchQuery.serialize), rankingMode = t.ThriftSearchRankingMode.Relevance, numResults = maxSearchResults, resultMetadataOptions = Some(metadataOptions), searcherId = query.getOptionalUserId, relevanceOptions = Some(relevanceOptions), maxHitsPerUser = -1, collectorParams = Some(collectorParams) ), clientRequestID = Some(s"${Trace.id.traceId}"), numResultsToReturnAtRoot = Some(maxSearchResults), clientId = Some(clientId.name), ) } private def createEntityIdsQuery(entityIds: Set[Long]): Disjunction = { val entityIdStrings = entityIds.map(_.toString) val queryOps: Seq[SearchQuery] = entityIdStrings.map { entityId => new SearchOperator(SearchOperator.Type.ENTITY_ID, entityId) }.toSeq new Disjunction(queryOps.asJava) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdFrsQueryTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird import com.twitter.conversions.DurationOps._ import com.twitter.core_workflows.user_model.{thriftscala => um} import com.twitter.home_mixer.functional_component.feature_hydrator.FrsSeedUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdFrsQueryTransformer._ import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.timelines.common.model.TweetKindOption object EarlybirdFrsQueryTransformer { private val DefaultSinceDuration = 24.hours private val ExpandedSinceDuration = 48.hours private val MaxTweetsToFetch = 500 private val TweetKindOptions: TweetKindOption.ValueSet = TweetKindOption(includeOriginalTweetsAndQuotes = true) private val UserStatesForExtendedSinceDuration: Set[um.UserState] = Set( um.UserState.Light, um.UserState.MediumNonTweeter, um.UserState.MediumTweeter, um.UserState.NearZero, um.UserState.New, um.UserState.VeryLight ) } case class EarlybirdFrsQueryTransformer[ Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext ]( candidatePipelineIdentifier: CandidatePipelineIdentifier, override val clientId: Option[String]) extends CandidatePipelineQueryTransformer[Query, eb.EarlybirdRequest] with EarlybirdQueryTransformer[Query] { override val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOptions override val maxTweetsToFetch: Int = MaxTweetsToFetch override def getTensorflowModel(query: Query): Option[String] = { Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.FrsParam)) } override def transform(query: Query): eb.EarlybirdRequest = { val userState = query.features.flatMap(_.getOrElse(UserStateFeature, None)) val sinceDuration = if (userState.exists(UserStatesForExtendedSinceDuration.contains)) ExpandedSinceDuration else DefaultSinceDuration val seedUserIds = query.features .flatMap(_.getOrElse(FrsSeedUserIdsFeature, None)) .getOrElse(Seq.empty).toSet buildEarlybirdQuery(query, sinceDuration, seedUserIds, None) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdInNetworkQueryTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird import com.twitter.conversions.DurationOps._ import com.twitter.core_workflows.user_model.{thriftscala => um} import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FollowedUserScoresFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdInNetworkQueryTransformer._ import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.timelines.common.model.TweetKindOption object EarlybirdInNetworkQueryTransformer { private val DefaultSinceDuration = 24.hours private val ExpandedSinceDuration = 48.hours private val MaxTweetsToFetch = 660 private val MaxFollowUsers = 1500 private val DefaultTweetKindOptions: TweetKindOption.ValueSet = TweetKindOption( includeReplies = true, includeRetweets = true, includeOriginalTweetsAndQuotes = true, includeExtendedReplies = true ) private val UserStatesForExtendedSinceDuration: Set[um.UserState] = Set( um.UserState.Light, um.UserState.MediumNonTweeter, um.UserState.MediumTweeter, um.UserState.NearZero, um.UserState.New, um.UserState.VeryLight ) } case class EarlybirdInNetworkQueryTransformer[ Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext ]( candidatePipelineIdentifier: CandidatePipelineIdentifier, override val clientId: Option[String]) extends CandidatePipelineQueryTransformer[Query, eb.EarlybirdRequest] with EarlybirdQueryTransformer[Query] { override def tweetKindOptions: TweetKindOption.ValueSet = DefaultTweetKindOptions override val maxTweetsToFetch: Int = MaxTweetsToFetch override val enableExcludeSourceTweetIdsQuery = true override def getTensorflowModel(query: Query): Option[String] = { Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.InNetworkParam)) } private def buildTweetKindOptions(query: Query): TweetKindOption.ValueSet = { TweetKindOption( includeReplies = query.params(ScoredTweetsParam.CandidateSourceParams.InNetworkIncludeRepliesParam), includeRetweets = query.params(ScoredTweetsParam.CandidateSourceParams.InNetworkIncludeRetweetsParam), includeOriginalTweetsAndQuotes = true, // Always include original tweets and quotes includeExtendedReplies = query.params(ScoredTweetsParam.CandidateSourceParams.InNetworkIncludeExtendedRepliesParam) ) } override def transform(query: Query): eb.EarlybirdRequest = { val userState = query.features.flatMap(_.getOrElse(UserStateFeature, None)) val sinceDuration = if (userState.exists(UserStatesForExtendedSinceDuration.contains)) ExpandedSinceDuration else DefaultSinceDuration val updatedAuthorScoreMap = query.features .map(_.getOrElse(FollowedUserScoresFeature, Map.empty[Long, Double])).toSeq.flatten.toMap val (authorScoreMap, followedUserIds) = if (updatedAuthorScoreMap.isEmpty) { ( query.features.map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double])), query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).toSeq.flatten.toSet) } else (Some(updatedAuthorScoreMap), updatedAuthorScoreMap.keySet) buildEarlybirdQueryWithTweetKindOptions( query, sinceDuration, followedUserIds.take(MaxFollowUsers) + query.getRequiredUserId, authorScoreMap, buildTweetKindOptions(query)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird/EarlybirdQueryTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.util.CachedScoredTweetsHelper import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.timelines.clients.relevance_search.SearchClient.TweetTypes import com.twitter.timelines.common.model.TweetKindOption import com.twitter.timelines.util.SnowflakeSortIndexHelper import com.twitter.util.Duration import com.twitter.util.Time trait EarlybirdQueryTransformer[ Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext] { def candidatePipelineIdentifier: CandidatePipelineIdentifier def clientId: Option[String] = None def maxTweetsToFetch: Int = 100 def tweetKindOptions: TweetKindOption.ValueSet def getTensorflowModel(query: Query): Option[String] = None def enableExcludeSourceTweetIdsQuery: Boolean = false private val EarlybirdMaxExcludedTweets = 1500 protected def getFollowedUsers(query: Query): Set[Long] = { query.features .map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty)).getOrElse( Nil).toSet + query.getRequiredUserId } protected def buildEarlybirdQuery( query: Query, sinceDuration: Duration, queryUserIds: Set[Long] = Set.empty, authorScoreMap: Option[Map[Long, Double]] = None, isVideoOnlyRequest: Boolean = false, getOlderTweets: Boolean = false, isRecency: Boolean = false, until: Time = Time.now ): eb.EarlybirdRequest = { buildEarlybirdQueryWithTweetKindOptions( query, sinceDuration, queryUserIds, authorScoreMap, tweetKindOptions, isVideoOnlyRequest, getOlderTweets, isRecency, until ) } protected def buildEarlybirdQueryWithTweetKindOptions( query: Query, sinceDuration: Duration, queryUserIds: Set[Long] = Set.empty, authorScoreMap: Option[Map[Long, Double]] = None, tweetKindOptions: TweetKindOption.ValueSet, isVideoOnlyRequest: Boolean = false, getOlderTweets: Boolean = false, isRecency: Boolean = false, until: Time = Time.now ): eb.EarlybirdRequest = { val sinceTime: Time = sinceDuration.ago val untilTime: Time = until val fromTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(sinceTime) val toTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(untilTime) val excludedTweetIds = query.features.map { featureMap => CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweetsInRange( featureMap, candidatePipelineIdentifier, EarlybirdMaxExcludedTweets, sinceTime, untilTime) } EarlybirdRequestUtil.getTweetsRequest( userId = Some(query.getRequiredUserId), clientId = clientId, skipVeryRecentTweets = true, queryUserIds = queryUserIds, retweetsMutedUserIds = Set.empty, beforeTweetIdExclusive = Some(toTweetIdExclusive), afterTweetIdExclusive = Some(fromTweetIdExclusive), excludedTweetIds = excludedTweetIds.map(_.toSet), maxCount = maxTweetsToFetch, tweetTypes = TweetTypes.fromTweetKindOption(tweetKindOptions), authorScoreMap = authorScoreMap, tensorflowModel = getTensorflowModel(query), enableExcludeSourceTweetIdsQuery = enableExcludeSourceTweetIdsQuery, isVideoOnlyRequest = isVideoOnlyRequest, getOlderTweets = getOlderTweets, isRecency = isRecency, ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/location", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", "src/thrift/com/twitter/timelineranker:thrift-scala", "topic-social-proof/server/src/main/thrift:thrift-scala", "tweet-mixer/thrift/src/main/thrift:thrift-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/CachedScoredTweetsResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.home_mixer.marshaller.timelines.TopicContextFunctionalityTypeUnmarshaller import com.twitter.home_mixer.model.candidate_source.SourceSignal import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorFollowersFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIsCreatorFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIsGoldVerifiedFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIsGrayVerifiedFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIsLegacyVerifiedFeature import com.twitter.home_mixer.model.HomeFeatures.CachedCandidatePipelineIdentifierFeature import com.twitter.home_mixer.model.HomeFeatures.ClipImageClusterIdsFeature import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.GorkContentCreatorFeature import com.twitter.home_mixer.model.HomeFeatures.GrokAnnotationsFeature import com.twitter.home_mixer.model.HomeFeatures.GrokContentCreatorFeature import com.twitter.home_mixer.model.HomeFeatures.GrokSlopScoreFeature import com.twitter.home_mixer.model.HomeFeatures.GrokTagsFeature import com.twitter.home_mixer.model.HomeFeatures.GrokTopCategoryFeature import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsArticleFeature import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.LastScoredTimestampMsFeature import com.twitter.home_mixer.model.HomeFeatures.ListIdFeature import com.twitter.home_mixer.model.HomeFeatures.ListNameFeature import com.twitter.home_mixer.model.HomeFeatures.MultiModalEmbeddingsFeature import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SourceSignalFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature import com.twitter.home_mixer.model.HomeFeatures.TweetMediaClusterIdsFeature import com.twitter.home_mixer.model.HomeFeatures.TweetMediaCompletionRateFeature import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature import com.twitter.home_mixer.model.HomeFeatures.TweetMixerScoreFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTypeMetricsFeature import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature import com.twitter.home_mixer.model.HomeFeatures.VideoDurationMsFeature import com.twitter.home_mixer.model.HomeFeatures.ViralContentCreatorFeature import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature import com.twitter.home_mixer.model.PredictedFavoriteScoreFeature import com.twitter.home_mixer.model.PredictedReplyScoreFeature import com.twitter.home_mixer.model.PredictedRetweetScoreFeature import com.twitter.home_mixer.model.PredictedShareScoreFeature import com.twitter.home_mixer.model.PredictedVideoQualityViewScoreFeature import com.twitter.home_mixer.model.PredictedDwellScoreFeature import com.twitter.home_mixer.model.PredictedNegativeFeedbackV2ScoreFeature import com.twitter.home_mixer.model.PredictedGoodClickConvoDescUamGt2ScoreFeature import com.twitter.home_mixer.model.PredictedGoodProfileClickScoreFeature import com.twitter.home_mixer.model.PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature import com.twitter.home_mixer.model.PredictedReplyEngagedByAuthorScoreFeature import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityNameFeature import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationIdFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier object CachedScoredTweetsResponseFeatureTransformer extends CandidateFeatureTransformer[hmt.ScoredTweet] { override val identifier: TransformerIdentifier = TransformerIdentifier("CachedScoredTweetsResponse") override val features: Set[Feature[_, _]] = Set( AncestorsFeature, AuthorIdFeature, AuthorIsBlueVerifiedFeature, AuthorIsCreatorFeature, AuthorIsGoldVerifiedFeature, AuthorIsGrayVerifiedFeature, AuthorIsLegacyVerifiedFeature, AuthorFollowersFeature, CachedCandidatePipelineIdentifierFeature, DirectedAtUserIdFeature, DebugStringFeature, ExclusiveConversationAuthorIdFeature, InNetworkFeature, InReplyToTweetIdFeature, InReplyToUserIdFeature, IsReadFromCacheFeature, IsRetweetFeature, LastScoredTimestampMsFeature, PredictionRequestIdFeature, QuotedTweetIdFeature, QuotedUserIdFeature, SGSValidFollowedByUserIdsFeature, SGSValidLikedByUserIdsFeature, ScoreFeature, SourceTweetIdFeature, SourceUserIdFeature, ServedTypeFeature, TopicContextFunctionalityTypeFeature, TopicIdSocialContextFeature, TweetTypeMetricsFeature, TweetUrlsFeature, ViralContentCreatorFeature, GrokContentCreatorFeature, GorkContentCreatorFeature, WeightedModelScoreFeature, CommunityIdFeature, CommunityNameFeature, ListIdFeature, ListNameFeature, LocationIdFeature, IsArticleFeature, HasVideoFeature, VideoDurationMsFeature, TweetMediaIdsFeature, GrokAnnotationsFeature, GrokTopCategoryFeature, GrokTagsFeature, PredictedFavoriteScoreFeature, PredictedReplyScoreFeature, PredictedRetweetScoreFeature, PredictedReplyEngagedByAuthorScoreFeature, PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, PredictedGoodClickConvoDescUamGt2ScoreFeature, PredictedGoodProfileClickScoreFeature, PredictedVideoQualityViewScoreFeature, PredictedShareScoreFeature, PredictedDwellScoreFeature, PredictedNegativeFeedbackV2ScoreFeature, TweetMixerScoreFeature, SourceSignalFeature, TweetMediaClusterIdsFeature, ClipImageClusterIdsFeature, GrokSlopScoreFeature, TweetMediaCompletionRateFeature, TweetTextFeature, MultiModalEmbeddingsFeature ) override def transform(candidate: hmt.ScoredTweet): FeatureMap = { val grokTopCategory = candidate.grokAnnotations .flatMap(_.categoryScores) .flatMap { scores => val validCategories = scores.collect { case (category, score) if category.forall(_.isDigit) && category.toLong % 10000 == 0 => (category.toLong, score) } if (validCategories.nonEmpty) Some(validCategories.maxBy(_._2)._1) else None } FeatureMapBuilder() .add(AncestorsFeature, candidate.ancestors.getOrElse(Seq.empty)) .add(AuthorIdFeature, Some(candidate.authorId)) .add(AuthorIsBlueVerifiedFeature, candidate.authorMetadata.exists(_.blueVerified)) .add(AuthorIsGoldVerifiedFeature, candidate.authorMetadata.exists(_.goldVerified)) .add(AuthorIsGrayVerifiedFeature, candidate.authorMetadata.exists(_.grayVerified)) .add(AuthorIsLegacyVerifiedFeature, candidate.authorMetadata.exists(_.legacyVerified)) .add(AuthorIsCreatorFeature, candidate.authorMetadata.exists(_.creator)) .add(AuthorFollowersFeature, candidate.authorMetadata.flatMap(_.followers)) .add(CachedCandidatePipelineIdentifierFeature, candidate.candidatePipelineIdentifier) .add(DirectedAtUserIdFeature, candidate.directedAtUserId) .add(DebugStringFeature, candidate.debugString) .add(ExclusiveConversationAuthorIdFeature, candidate.exclusiveConversationAuthorId) .add(InNetworkFeature, candidate.inNetwork.getOrElse(true)) .add(InReplyToTweetIdFeature, candidate.inReplyToTweetId) .add(InReplyToUserIdFeature, candidate.inReplyToUserId) .add(IsReadFromCacheFeature, true) .add(IsRetweetFeature, candidate.sourceTweetId.isDefined) .add(LastScoredTimestampMsFeature, candidate.lastScoredTimestampMs) .add(PredictionRequestIdFeature, candidate.predictionRequestId) .add(QuotedTweetIdFeature, candidate.quotedTweetId) .add(QuotedUserIdFeature, candidate.quotedUserId) .add(ScoreFeature, candidate.score) .add(SGSValidLikedByUserIdsFeature, candidate.sgsValidLikedByUserIds.getOrElse(Seq.empty)) .add( SGSValidFollowedByUserIdsFeature, candidate.sgsValidFollowedByUserIds.getOrElse(Seq.empty)) .add(SourceTweetIdFeature, candidate.sourceTweetId) .add(SourceUserIdFeature, candidate.sourceUserId) .add(ServedTypeFeature, candidate.servedType) .add( TopicContextFunctionalityTypeFeature, candidate.topicFunctionalityType.map(TopicContextFunctionalityTypeUnmarshaller(_))) .add(TopicIdSocialContextFeature, candidate.topicId) .add(TweetTypeMetricsFeature, candidate.tweetTypeMetrics) .add(TweetUrlsFeature, candidate.tweetUrls.getOrElse(Seq.empty)) .add(ViralContentCreatorFeature, candidate.viralContentCreator.contains(true)) .add(WeightedModelScoreFeature, candidate.score) .add(CommunityIdFeature, candidate.communityId) .add(CommunityNameFeature, candidate.communityName) .add(ListIdFeature, candidate.listId) .add(ListNameFeature, candidate.listName) .add(LocationIdFeature, candidate.locationId) .add(IsArticleFeature, candidate.isArticle.contains(true)) .add(HasVideoFeature, candidate.hasVideo.contains(true)) .add(VideoDurationMsFeature, candidate.videoDurationMs) .add(TweetMediaIdsFeature, candidate.mediaIds.getOrElse(Seq.empty)) .add(GrokAnnotationsFeature, candidate.grokAnnotations) .add(GrokTopCategoryFeature, grokTopCategory) .add( GrokTagsFeature, candidate.grokAnnotations.map(_.tags.map(_.toLowerCase)).getOrElse(Seq.empty).toSet ) .add(PredictedFavoriteScoreFeature, candidate.predictedScores.flatMap(_.favoriteScore)) .add(PredictedReplyScoreFeature, candidate.predictedScores.flatMap(_.replyScore)) .add(PredictedRetweetScoreFeature, candidate.predictedScores.flatMap(_.retweetScore)) .add( PredictedReplyEngagedByAuthorScoreFeature, candidate.predictedScores.flatMap(_.replyEngagedByAuthorScore)) .add( PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, candidate.predictedScores.flatMap(_.goodClickConvoDescFavoritedOrRepliedScore)) .add( PredictedGoodClickConvoDescUamGt2ScoreFeature, candidate.predictedScores.flatMap(_.goodClickConvoDescUamGt2Score)) .add( PredictedGoodProfileClickScoreFeature, candidate.predictedScores.flatMap(_.goodProfileClickScore)) .add( PredictedVideoQualityViewScoreFeature, candidate.predictedScores.flatMap(_.videoQualityViewScore)) .add(PredictedShareScoreFeature, candidate.predictedScores.flatMap(_.shareScore)) .add(PredictedDwellScoreFeature, candidate.predictedScores.flatMap(_.dwellScore)) .add( PredictedNegativeFeedbackV2ScoreFeature, candidate.predictedScores.flatMap(_.negativeFeedbackV2Score)) .add(TweetMixerScoreFeature, candidate.tweetMixerScore) .add( SourceSignalFeature, candidate.sourceSignal.map { signal => SourceSignal( id = signal.id, signalType = signal.signalType, signalEntity = signal.signalEntity, authorId = signal.authorId, ) } ) .add( ClipImageClusterIdsFeature, candidate.clipClusterIdsFeature .flatMap(_.clipImageClusterIdsFeature).getOrElse(Map.empty[Long, Long]).toMap) .add( TweetMediaClusterIdsFeature, candidate.clipClusterIdsFeature .flatMap(_.tweetMediaClusterIdsFeature).getOrElse(Map.empty[Long, Long]).toMap) .add(GrokSlopScoreFeature, candidate.grokSlopScoreFeature) .add(TweetMediaCompletionRateFeature, candidate.mediaCompletionRate) .add(TweetTextFeature, candidate.tweetText) .add(GrokContentCreatorFeature, candidate.grokContentCreator.contains(true)) .add(GorkContentCreatorFeature, candidate.gorkContentCreator.contains(true)) .add(MultiModalEmbeddingsFeature, candidate.multiModalEmbedding) .build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsBackfillResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier object ScoredTweetsBackfillResponseFeatureTransformer extends CandidateFeatureTransformer[Long] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsBackfillResponse") override val features: Set[Feature[_, _]] = Set( FromInNetworkSourceFeature, ServedTypeFeature, DebugStringFeature ) override def transform(candidate: Long): FeatureMap = FeatureMapBuilder() .add(FromInNetworkSourceFeature, true) .add(ServedTypeFeature, hmt.ServedType.ForYouInNetwork) .add(DebugStringFeature, Some("Backfill")) .build() } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsContentExplorationResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.product.scored_tweets.candidate_source.ContentExplorationCandidateResponse import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier object ScoredTweetsContentExplorationResponseFeatureTransformer extends CandidateFeatureTransformer[ContentExplorationCandidateResponse] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsContentExploration") override val features: Set[Feature[_, _]] = Set( ServedTypeFeature, DebugStringFeature ) override def transform(candidate: ContentExplorationCandidateResponse): FeatureMap = { val servedType = candidate.tier match { case "tier1" => hmt.ServedType.ForYouContentExploration case "tier2" => hmt.ServedType.ForYouContentExplorationTier2 case _ => hmt.ServedType.ForYouContentExploration } FeatureMapBuilder() .add(ServedTypeFeature, servedType) .add(DebugStringFeature, Some(s"Content Exploration: ${candidate.tier}")) .build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsDirectUtegResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.recos.recos_common.thriftscala.SocialProofType import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} object UtegFavListFeature extends Feature[TweetCandidate, Seq[Long]] object UtegScoreFeature extends Feature[TweetCandidate, Double] object ScoredTweetsDirectUtegResponseFeatureTransformer extends CandidateFeatureTransformer[uteg.TweetRecommendation] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsDirectUtegResponse") override val features: Set[Feature[_, _]] = Set( FromInNetworkSourceFeature, ServedTypeFeature, DebugStringFeature, UtegFavListFeature, UtegScoreFeature ) override def transform(input: uteg.TweetRecommendation): FeatureMap = FeatureMapBuilder() .add(FromInNetworkSourceFeature, false) .add(ServedTypeFeature, hmt.ServedType.ForYouUteg) .add(DebugStringFeature, Some("Uteg")) .add( UtegFavListFeature, input.socialProofByType.getOrElse(SocialProofType.Favorite, Seq.empty) ) .add(UtegScoreFeature, input.score) .build() } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsFrsResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.timelineranker.{thriftscala => tlr} object ScoredTweetsFrsResponseFeatureTransformer extends CandidateFeatureTransformer[tlr.CandidateTweet] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsFrsResponse") override val features: Set[Feature[_, _]] = TimelineRankerResponseTransformer.features override def transform(candidate: tlr.CandidateTweet): FeatureMap = { val baseFeatures = TimelineRankerResponseTransformer.transform(candidate) val features = FeatureMapBuilder() .add(ServedTypeFeature, hmt.ServedType.ForYouFrs) .build() baseFeatures ++ features } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsInNetworkResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SourceSignalFeature import com.twitter.home_mixer.model.candidate_source.SourceSignal import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.timelineranker.{thriftscala => tlr} object ScoredTweetsInNetworkResponseFeatureTransformer extends CandidateFeatureTransformer[tlr.CandidateTweet] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsInNetworkResponse") override val features: Set[Feature[_, _]] = TimelineRankerResponseTransformer.features ++ Set(SourceSignalFeature) override def transform(candidate: tlr.CandidateTweet): FeatureMap = { val baseFeatures = TimelineRankerResponseTransformer.transform(candidate) val features = FeatureMapBuilder() .add(FromInNetworkSourceFeature, true) .add(ServedTypeFeature, hmt.ServedType.ForYouInNetwork) .add( SourceSignalFeature, Some( SourceSignal( id = candidate.tweet.flatMap(_.coreData.map(_.userId)).getOrElse(0L), signalType = None, signalEntity = None, authorId = None, )) ) .build() baseFeatures ++ features } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsListsResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.ListIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.product.scored_tweets.candidate_source.ListTweet import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier object ScoredTweetsListsResponseFeatureTransformer extends CandidateFeatureTransformer[ListTweet] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsListsResponse") override val features: Set[Feature[_, _]] = Set( AuthorIdFeature, FromInNetworkSourceFeature, IsRetweetFeature, ServedTypeFeature, SourceTweetIdFeature, SourceUserIdFeature, ListIdFeature ) override def transform(candidate: ListTweet): FeatureMap = FeatureMapBuilder() .add(AuthorIdFeature, candidate.tweet.userId) .add(FromInNetworkSourceFeature, false) .add(IsRetweetFeature, candidate.tweet.sourceStatusId.isDefined) .add(ServedTypeFeature, hmt.ServedType.ForYouList) .add(SourceTweetIdFeature, candidate.tweet.sourceStatusId) .add(SourceUserIdFeature, candidate.tweet.sourceUserId) .add(ListIdFeature, Some(candidate.listId)) .build() } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsOfflineVideoRecoResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier object ScoredTweetsOfflineVideoRecoResponseFeatureTransformer extends CandidateFeatureTransformer[Long] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsOfflineVideoRecoResponse") override val features: Set[Feature[_, _]] = Set( FromInNetworkSourceFeature, ServedTypeFeature ) override def transform(candidate: Long): FeatureMap = FeatureMapBuilder() .add(FromInNetworkSourceFeature, false) .add(ServedTypeFeature, hmt.ServedType.OfflineVideoReco) .build() } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsPopularVideosResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.explore_ranker.{thriftscala => ert} import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetFeature import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} import com.twitter.timelineservice.suggests.{thriftscala => st} object ScoredTweetsPopularVideosResponseFeatureTransformer extends CandidateFeatureTransformer[ert.ExploreTweetRecommendation] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsPopularVideosResponse") override val features: Set[Feature[_, _]] = Set( AuthorIdFeature, CandidateSourceIdFeature, FromInNetworkSourceFeature, HasVideoFeature, IsRandomTweetFeature, StreamToKafkaFeature, SuggestTypeFeature ) override def transform(candidate: ert.ExploreTweetRecommendation): FeatureMap = { FeatureMapBuilder() .add(AuthorIdFeature, candidate.authorId) .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.MediaTweet)) .add(FromInNetworkSourceFeature, false) .add(HasVideoFeature, candidate.mediaType.contains(ert.MediaType.Video)) .add(IsRandomTweetFeature, false) .add(StreamToKafkaFeature, true) .add(SuggestTypeFeature, Some(st.SuggestType.MediaTweet)) .build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsStaticResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier object ScoredTweetsStaticResponseFeatureTransformer extends CandidateFeatureTransformer[Long] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsStatic") override val features: Set[Feature[_, _]] = Set( FromInNetworkSourceFeature, ServedTypeFeature, ScoreFeature, ) private val StaticScore = 100000.0 override def transform(candidate: Long): FeatureMap = { FeatureMapBuilder() .add(FromInNetworkSourceFeature, false) .add(ServedTypeFeature, hmt.ServedType.ForYouTweetMixer) .add(ScoreFeature, Some(StaticScore)) .build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsTweetMixerResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.model.candidate_source._ import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.tsp.{thriftscala => tsp} import com.twitter.tweet_mixer.{thriftscala => tmt} import com.twitter.usersignalservice.{thriftscala => uss} case class ScoredTweetsTweetMixerResponseFeatureTransformer(debugPrefix: String = "") extends CandidateFeatureTransformer[tmt.TweetResult] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsTweetMixerResponse") override val features: Set[Feature[_, _]] = Set( FromInNetworkSourceFeature, ServedTypeFeature, TSPMetricTagFeature, TweetMixerScoreFeature, InReplyToTweetIdFeature, DebugStringFeature, SourceSignalFeature, ) val FavoriteSignal = "Fav" val RetweetSignal = "Retweet" val ReplySignal = "Reply" val BookmarkSignal = "Bookmark" val ShareSignal = "Share" val TweetSignal = "Tweet" val VideoViewSignal = "VideoView" val ImmersiveVideoViewSignal = "ImmersiveVideoView" val SearcherRealtimeHistorySignal = "SearcherRealtimeHistory" val FollowSignal = "Follow" val ProfileVisitSignal = "ProfileVisit" val NotificationsSignal = "Notif" val UTEG = "UTEG" val PopGeo = "PopGeo" val PopTopic = "PopTopic" val Simclusters = "Simclusters" val Twhin = "Twhin" val UTG = "UTG" val UVG = "UVG" val InNetwork = "InNetwork" val DeepRetrieval = "DeepRetrieval" val DeepRetrievalI2i = "DeepRetrievalI2i" val ContentExploration = "Tier1ContentExploration" val ContentExplorationTier2 = "Tier2ContentExploration" val ContentExplorationDeepRetrievalI2i = "Tier1DrContentExploration" val ContentExplorationTier2DeepRetrievalI2i = "Tier2DrContentExploration" val ContentExplorationSimclusterColdPosts = "ContentExplorationSimclusterColdPosts" val EvergreenDeepRetrievalHome = "EvergreenDeepRetrievalHome" val EvergreenDeepRetrievalCrossBorderHome = "EvergreenDeepRetrievalCrossBorderHome" val UserInterestSummary = "UserInterestSummary" val ContentExplorationEvergreenDRI2i = "ContentExplorationEvergreenDRI2i" val Local = "Local" val Trends = "Trends" val TwitterClipV0Short = "TwitterClipV0Short" val TwitterClipV0Long = "TwitterClipV0Long" val SemanticVideo = "SemanticVideo" val RelatedCreator = "RelatedCreator" override def transform(candidate: tmt.TweetResult): FeatureMap = { val tweetMixerMetricTags: Seq[tmt.MetricTag] = candidate.metricTags.getOrElse(Seq.empty) val tspMetricTag = tweetMixerMetricTags .map(tweetMixerMetricTagToTspMetricTag) .filter(_.nonEmpty).map(_.get).toSet val servedType: hmt.ServedType = getServedType(candidate.tweetMetadata) val fromInNetwork = servedType match { case hmt.ServedType.ForYouInNetwork => true case _ => false } FeatureMapBuilder() .add(FromInNetworkSourceFeature, fromInNetwork) .add(ServedTypeFeature, servedType) .add(TSPMetricTagFeature, tspMetricTag) .add(TweetMixerScoreFeature, candidate.score) .add(DebugStringFeature, candidate.tweetMetadata.map(buildDebugString)) .add(SourceSignalFeature, candidate.tweetMetadata.map(buildSourceSignal)) .add(InReplyToTweetIdFeature, candidate.inReplyToTweetId) .build() } private def tweetMixerMetricTagToTspMetricTag( tweetMixerMetricTag: tmt.MetricTag ): Option[tsp.MetricTag] = tweetMixerMetricTag match { case tmt.MetricTag.TweetFavorite => Some(tsp.MetricTag.TweetFavorite) case tmt.MetricTag.Retweet => Some(tsp.MetricTag.Retweet) case tmt.MetricTag.PopGeo => None case _ => None } private def buildSourceSignal(metadata: tmt.TweetMetadata): SourceSignal = { SourceSignal( id = metadata.sourceSignalId.getOrElse(0L), signalType = metadata.signalType.flatMap(_.headOption.map(_.name)), signalEntity = metadata.signalEntity, authorId = metadata.authorId, ) } private def buildDebugString(metadata: tmt.TweetMetadata): String = { val signalTypeStr = metadata.signalType .map { signalTypes => signalTypes.map { case uss.SignalType.TweetFavorite => FavoriteSignal case uss.SignalType.Retweet => RetweetSignal case uss.SignalType.Reply => ReplySignal case uss.SignalType.TweetBookmarkV1 => BookmarkSignal case uss.SignalType.TweetShareV1 => ShareSignal case uss.SignalType.OriginalTweet => TweetSignal case uss.SignalType.VideoView90dQualityV1 | uss.SignalType.VideoView90dPlayback50V1 | uss.SignalType.VideoView90dQualityV1AllSurfaces => VideoViewSignal case uss.SignalType.ImmersiveVideoQualityView => ImmersiveVideoViewSignal case uss.SignalType.SearcherRealtimeHistory => SearcherRealtimeHistorySignal case uss.SignalType.AccountFollow => FollowSignal case uss.SignalType.RepeatedProfileVisit180dMinVisit6V1 | uss.SignalType.RepeatedProfileVisit90dMinVisit6V1 | uss.SignalType.RepeatedProfileVisit14dMinVisit2V1 | uss.SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative | uss.SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative | uss.SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative => ProfileVisitSignal case uss.SignalType.NotificationOpenAndClickV1 => NotificationsSignal case other => other.name } }.getOrElse(Seq.empty).mkString(",") val candidateTypeStr = metadata.servedType .map { case tmt.ServedType.Simclusters => Simclusters case tmt.ServedType.Twhin => Twhin case tmt.ServedType.Utg => UTG case tmt.ServedType.Uvg => UVG case tmt.ServedType.Uteg => UTEG case tmt.ServedType.InNetwork => InNetwork case tmt.ServedType.PopGeo => PopGeo case tmt.ServedType.PopTopic => PopTopic case tmt.ServedType.DeepRetrieval => DeepRetrieval case tmt.ServedType.DeepRetrievalI2iEmb => DeepRetrievalI2i case tmt.ServedType.ContentExploration => ContentExploration case tmt.ServedType.ContentExplorationTier2 => ContentExplorationTier2 case tmt.ServedType.ContentExplorationDRI2i => ContentExplorationDeepRetrievalI2i case tmt.ServedType.ContentExplorationDRI2iTier2 => ContentExplorationTier2DeepRetrievalI2i case tmt.ServedType.ContentExplorationSimclusterColdPosts => ContentExplorationSimclusterColdPosts case tmt.ServedType.EvergreenDRU2iHome => EvergreenDeepRetrievalHome case tmt.ServedType.EvergreenDRCrossBorderU2iHome => EvergreenDeepRetrievalCrossBorderHome case tmt.ServedType.UserInterestSummaryI2i => UserInterestSummary case tmt.ServedType.ContentExplorationEvergreenDRI2i => ContentExplorationEvergreenDRI2i case tmt.ServedType.Local => Local case tmt.ServedType.Trends => Trends case tmt.ServedType.TwitterClipV0Short => TwitterClipV0Short case tmt.ServedType.TwitterClipV0Long => TwitterClipV0Long case tmt.ServedType.SemanticVideo => SemanticVideo case tmt.ServedType.RelatedCreator => RelatedCreator case _ => "" }.getOrElse("") s"${metadata.sourceSignalId.getOrElse(0L)} $signalTypeStr $candidateTypeStr $debugPrefix" } private def getServedType(metadata: Option[tmt.TweetMetadata]): hmt.ServedType = { metadata .flatMap { _.servedType .map { case tmt.ServedType.Simclusters => hmt.ServedType.ForYouSimclusters case tmt.ServedType.Twhin => hmt.ServedType.ForYouTwhin case tmt.ServedType.Utg => hmt.ServedType.ForYouUtg case tmt.ServedType.Uvg => hmt.ServedType.ForYouUvg case tmt.ServedType.Uteg => hmt.ServedType.ForYouUteg case tmt.ServedType.InNetwork => hmt.ServedType.ForYouInNetwork case tmt.ServedType.PopGeo => hmt.ServedType.ForYouPopularGeo case tmt.ServedType.PopTopic => hmt.ServedType.ForYouPopularTopic case tmt.ServedType.DeepRetrieval => hmt.ServedType.ForYouDeepRetrieval case tmt.ServedType.DeepRetrievalI2iEmb => hmt.ServedType.ForYouDeepRetrievalI2i case tmt.ServedType.ContentExploration => hmt.ServedType.ForYouContentExploration case tmt.ServedType.ContentExplorationTier2 => hmt.ServedType.ForYouContentExplorationTier2 case tmt.ServedType.ContentExplorationDRI2i => hmt.ServedType.ForYouContentExplorationDeepRetrievalI2i case tmt.ServedType.ContentExplorationDRI2iTier2 => hmt.ServedType.ForYouContentExplorationTier2DeepRetrievalI2i case tmt.ServedType.EvergreenDeepRetrieval => hmt.ServedType.ForYouEvergreenDeepRetrieval case tmt.ServedType.EvergreenDRU2iHome => hmt.ServedType.ForYouEvergreenDeepRetrievalHome case tmt.ServedType.EvergreenDRCrossBorderU2iHome => hmt.ServedType.ForYouEvergreenDeepRetrievalCrossBorderHome case tmt.ServedType.UserInterestSummaryI2i => hmt.ServedType.ForYouUserInterestSummary case tmt.ServedType.ContentExplorationEvergreenDRI2i => hmt.ServedType.ForYouContentExplorationEvergreenDeepRetrievalI2i case tmt.ServedType.ContentExplorationSimclusterColdPosts => hmt.ServedType.ForYouContentExplorationSimclusterColdPosts case tmt.ServedType.Local => hmt.ServedType.ForYouLocal case tmt.ServedType.Trends => hmt.ServedType.ForYouTrends case tmt.ServedType.TwitterClipV0Short => hmt.ServedType.ForYouTwitterClipV0Short case tmt.ServedType.TwitterClipV0Long => hmt.ServedType.ForYouTwitterClipV0Long case tmt.ServedType.SemanticVideo => hmt.ServedType.ForYouSemanticVideo case tmt.ServedType.RelatedCreator => hmt.ServedType.ForYouRelatedCreator case tmt.ServedType.PromotedCreator => hmt.ServedType.ForYouPromotedCreator case tmt.ServedType.NsfwVideoContent => hmt.ServedType.ForYouNsfwVideoContent case _ => hmt.ServedType.ForYouTweetMixer } }.getOrElse(hmt.ServedType.ForYouTweetMixer) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredTweetsUtegResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.timelineranker.{thriftscala => tlr} import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts} import com.twitter.timelineservice.suggests.{thriftscala => st} object ScoredTweetsUtegResponseFeatureTransformer extends CandidateFeatureTransformer[tlr.CandidateTweet] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsUtegResponse") override val features: Set[Feature[_, _]] = TimelineRankerResponseTransformer.features override def transform(candidate: tlr.CandidateTweet): FeatureMap = { val baseFeatures = TimelineRankerResponseTransformer.transform(candidate) val features = FeatureMapBuilder() .add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.RecommendedTweet)) .add(SuggestTypeFeature, Some(st.SuggestType.ActivityTweet)) .build() baseFeatures ++ features } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/ScoredVideoTweetsPinnedTweetResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier object ScoredVideoTweetsPinnedTweetResponseFeatureTransformer extends CandidateFeatureTransformer[TweetCandidate] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredVideoTweetsPinnedTweetResponse") override val features: Set[Feature[_, _]] = Set( ServedTypeFeature ) override def transform(candidate: TweetCandidate): FeatureMap = FeatureMapBuilder() .add(ServedTypeFeature, hmt.ServedType.ForYouPinned) .build() } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/TimelineRankerResponseTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature import com.twitter.home_mixer.util.tweetypie.content.TweetMediaFeaturesExtractor import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.timelineranker.{thriftscala => tlr} object TimelineRankerResponseTransformer { val features: Set[Feature[_, _]] = Set( AuthorIdFeature, CommunityIdFeature, DirectedAtUserIdFeature, EarlybirdFeature, EarlybirdScoreFeature, ExclusiveConversationAuthorIdFeature, FromInNetworkSourceFeature, HasImageFeature, HasVideoFeature, InReplyToTweetIdFeature, InReplyToUserIdFeature, IsRetweetFeature, MentionScreenNameFeature, MentionUserIdFeature, QuotedTweetIdFeature, QuotedUserIdFeature, SourceTweetIdFeature, SourceUserIdFeature, ServedTypeFeature, TweetUrlsFeature ) def transform(candidate: tlr.CandidateTweet): FeatureMap = { val tweet = candidate.tweet val quotedTweet = tweet.filter(_.quotedTweet.exists(_.tweetId != 0)).flatMap(_.quotedTweet) val mentions = tweet.flatMap(_.mentions).getOrElse(Seq.empty) val coreData = tweet.flatMap(_.coreData) val share = coreData.flatMap(_.share) val reply = coreData.flatMap(_.reply) val communityId = tweet.flatMap(_.communities).flatMap(_.communityIds.headOption) FeatureMapBuilder() .add(AuthorIdFeature, coreData.map(_.userId)) .add(CommunityIdFeature, communityId) .add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId))) .add(EarlybirdFeature, candidate.features) .add(EarlybirdScoreFeature, candidate.features.map(_.earlybirdScore)) .add( ExclusiveConversationAuthorIdFeature, tweet.flatMap(_.exclusiveTweetControl.map(_.conversationAuthorId))) .add(FromInNetworkSourceFeature, false) .add(HasImageFeature, tweet.exists(TweetMediaFeaturesExtractor.hasImage)) .add(HasVideoFeature, tweet.exists(TweetMediaFeaturesExtractor.hasVideo)) .add(InReplyToTweetIdFeature, reply.flatMap(_.inReplyToStatusId)) .add(InReplyToUserIdFeature, reply.map(_.inReplyToUserId)) .add(IsRetweetFeature, share.isDefined) .add(MentionScreenNameFeature, mentions.map(_.screenName)) .add(MentionUserIdFeature, mentions.flatMap(_.userId)) .add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId)) .add(QuotedUserIdFeature, quotedTweet.map(_.userId)) .add(SourceTweetIdFeature, share.map(_.sourceStatusId)) .add(SourceUserIdFeature, share.map(_.sourceUserId)) .add(TweetUrlsFeature, candidate.features.flatMap(_.urlsList).getOrElse(Seq.empty)) .build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", "src/thrift/com/twitter/search:earlybird-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/EarlybirdResponseTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdSearchResultFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.search.earlybird.{thriftscala => eb} object EarlybirdResponseTransformer { val features: Set[Feature[_, _]] = Set( EarlybirdScoreFeature, EarlybirdSearchResultFeature, InReplyToTweetIdFeature, InReplyToUserIdFeature, IsRetweetFeature, TweetUrlsFeature ) def transform(candidate: eb.ThriftSearchResult): FeatureMap = { val metadata = candidate.metadata val isRetweet = metadata.flatMap(_.isRetweet).getOrElse(false) val sharedStatusId = metadata.map(_.sharedStatusId).getOrElse(0L) val referencedTweetAuthorId = metadata.map(_.referencedTweetAuthorId).getOrElse(0L) val inReplyToTweetId = if (!isRetweet && sharedStatusId > 0) Some(sharedStatusId) else None val inReplyToUserId = if (!isRetweet && sharedStatusId > 0 && referencedTweetAuthorId > 0) Some(referencedTweetAuthorId) else None FeatureMapBuilder() .add(EarlybirdSearchResultFeature, Some(candidate)) .add(EarlybirdScoreFeature, candidate.metadata.flatMap(_.score)) .add(InReplyToTweetIdFeature, inReplyToTweetId) .add(InReplyToUserIdFeature, inReplyToUserId) .add(IsRetweetFeature, isRetweet) .add( TweetUrlsFeature, candidate.metadata.flatMap(_.tweetUrls.map(_.map(_.originalUrl))).getOrElse(Seq.empty)) .build() } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsCommunitiesResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.search.earlybird.{thriftscala => t} object ScoredTweetsCommunitiesResponseFeatureTransformer extends CandidateFeatureTransformer[t.ThriftSearchResult] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsEarlybirdCommunitiesResponse") private val candidateSourceFeatures = Set( FromInNetworkSourceFeature, ServedTypeFeature ) override val features: Set[Feature[_, _]] = EarlybirdResponseTransformer.features ++ Set(CommunityIdFeature) ++ candidateSourceFeatures override def transform(candidate: t.ThriftSearchResult): FeatureMap = { val baseFeatures = EarlybirdResponseTransformer.transform(candidate) val communityIdOpt = candidate.tweetypieTweet.flatMap(_.communities.flatMap(_.communityIds.headOption)) val features = FeatureMapBuilder() .add(FromInNetworkSourceFeature, false) .add(ServedTypeFeature, hmt.ServedType.ForYouCommunity) .add(CommunityIdFeature, communityIdOpt) .build() baseFeatures ++ features } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdFrsResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.search.earlybird.{thriftscala => eb} object ScoredTweetsEarlybirdFrsResponseFeatureTransformer extends CandidateFeatureTransformer[eb.ThriftSearchResult] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsEarlybirdFrsResponse") private val candidateSourceFeatures = Set( FromInNetworkSourceFeature, ServedTypeFeature, DebugStringFeature ) override val features: Set[Feature[_, _]] = EarlybirdResponseTransformer.features ++ candidateSourceFeatures override def transform(candidate: eb.ThriftSearchResult): FeatureMap = { val baseFeatures = EarlybirdResponseTransformer.transform(candidate) val features = FeatureMapBuilder() .add(FromInNetworkSourceFeature, false) .add(ServedTypeFeature, hmt.ServedType.ForYouFrs) .add(DebugStringFeature, Some("FRS")) .build() baseFeatures ++ features } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.search.earlybird.{thriftscala => eb} object ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer extends CandidateFeatureTransformer[eb.ThriftSearchResult] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsEarlybirdInNetworkResponse") private val candidateSourceFeatures = Set( FromInNetworkSourceFeature, ServedTypeFeature, DebugStringFeature ) override val features: Set[Feature[_, _]] = EarlybirdResponseTransformer.features ++ candidateSourceFeatures override def transform(candidate: eb.ThriftSearchResult): FeatureMap = { val baseFeatures = EarlybirdResponseTransformer.transform(candidate) val features = FeatureMapBuilder() .add(FromInNetworkSourceFeature, true) .add(ServedTypeFeature, hmt.ServedType.ForYouInNetwork) .add(DebugStringFeature, Some("In Network")) .build() baseFeatures ++ features } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird/ScoredTweetsEarlybirdNsfwResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird import com.twitter.home_mixer.model.HomeFeatures.DebugStringFeature import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.search.earlybird.{thriftscala => eb} object ScoredTweetsEarlybirdNsfwResponseFeatureTransformer extends CandidateFeatureTransformer[eb.ThriftSearchResult] { override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsEarlybirdNsfwResponse") private val candidateSourceFeatures = Set( FromInNetworkSourceFeature, ServedTypeFeature, DebugStringFeature ) override val features: Set[Feature[_, _]] = EarlybirdResponseTransformer.features ++ candidateSourceFeatures override def transform(candidate: eb.ThriftSearchResult): FeatureMap = { val baseFeatures = EarlybirdResponseTransformer.transform(candidate) val features = FeatureMapBuilder() .add(FromInNetworkSourceFeature, false) .add(ServedTypeFeature, hmt.ServedType.ForYouNsfwVideoContent) .add(DebugStringFeature, Some("Nsfw Video")) .build() baseFeatures ++ features } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/AuthorBasedListwiseRescoringProvider.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery object AuthorBasedListwiseRescoringProvider extends ListwiseRescoringProvider[CandidateWithFeatures[TweetCandidate], Long] { private val MinFollowed = 50 /** * Author-based, the groupBy key is the author id. * Rescore the list of candidates that share the same author id. */ override def groupByKey(candidate: CandidateWithFeatures[TweetCandidate]): Option[Long] = candidate.features.getOrElse(AuthorIdFeature, None) /** * Defines the list of author-based candidate rescorers. * Multiply the rescorers together for each candidate. */ override def candidateRescoringFactor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate], index: Int ): Double = { val isSmallFollowGraph = query.features.get.getOrElse(SGSFollowedUsersFeature, Seq.empty).size <= MinFollowed val decayFactor = if (isSmallFollowGraph) { query.params(ScoredTweetsParam.SmallFollowGraphAuthorDiversityDecayFactor) } else { query.params(ScoredTweetsParam.AuthorDiversityDecayFactor) } val floor = if (isSmallFollowGraph) query.params(ScoredTweetsParam.SmallFollowGraphAuthorDiversityFloor) else query.params(ScoredTweetsParam.AuthorDiversityFloor) authorDiversityBasedRescorer(index = index, decayFactor = decayFactor, floor = floor) } /** * Re-scoring multiplier to apply to multiple tweets from the same author. * Provides an exponential decay based discount by position (with a floor). */ def authorDiversityBasedRescorer( index: Int, decayFactor: Double, floor: Double ): Double = (1 - floor) * Math.pow(decayFactor, index) + floor } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/user_history", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/control_ai", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", "src/thrift/com/twitter/timelines/control_ai:timeline-control-ai-thrift-scala", "timelineservice/common:model", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/CandidateSourceDiversityListwiseRescoringProvider.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SourceSignalFeature import com.twitter.home_mixer.model.candidate_source.SourceSignal import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery object CandidateSourceDiversityListwiseRescoringProvider extends ListwiseRescoringProvider[ CandidateWithFeatures[TweetCandidate], (hmt.ServedType, Option[SourceSignal]) ] { override def groupByKey( candidate: CandidateWithFeatures[TweetCandidate] ): Option[(hmt.ServedType, Option[SourceSignal])] = { val servedType = candidate.features.get(ServedTypeFeature) val sourceSignalOpt = candidate.features.getOrElse(SourceSignalFeature, None) Some((servedType, sourceSignalOpt)) } override def candidateRescoringFactor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate], index: Int ): Double = candidate.features.get(ServedTypeFeature) match { case hmt.ServedType.ForYouInNetwork => 1.0 case _ => if (query.params(ScoredTweetsParam.EnableCandidateSourceDiversityDecay)) { val decayFactor = query.params(ScoredTweetsParam.CandidateSourceDiversityDecayFactor) val floor = query.params(ScoredTweetsParam.CandidateSourceDiversityFloor) candidateSourceDiversityRescorer(index = index, decayFactor = decayFactor, floor = floor) } else 1.0 } /** * Re-scoring multiplier to apply to multiple tweets from the same candidate source and reason. * Provides an exponential decay based discount by position (with a floor). */ def candidateSourceDiversityRescorer( index: Int, decayFactor: Double, floor: Double ): Double = (1 - floor) * Math.pow(decayFactor, index) + floor } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ContentExplorationListwiseRescoringProvider.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableContentExplorationCandidateMaxCountParam import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery object ContentExplorationListwiseRescoringProvider extends ListwiseRescoringProvider[CandidateWithFeatures[TweetCandidate], hmt.ServedType] { override def groupByKey( candidate: CandidateWithFeatures[TweetCandidate] ): Option[hmt.ServedType] = Some(candidate.features.get(ServedTypeFeature)) override def candidateRescoringFactor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate], index: Int ): Double = { if (query.params(EnableContentExplorationCandidateMaxCountParam)) { val servedType = candidate.features.get(ServedTypeFeature) if (servedType == hmt.ServedType.ForYouUserInterestSummary || servedType == hmt.ServedType.ForYouContentExploration || servedType == hmt.ServedType.ForYouContentExplorationTier2 || servedType == hmt.ServedType.ForYouContentExplorationDeepRetrievalI2i || servedType == hmt.ServedType.ForYouContentExplorationTier2DeepRetrievalI2i) 0.0001 else 1.0 } else 1.0 } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Map[Long, Double] = { candidates .groupBy(groupByKey) .flatMap { case (Some(servedType), groupedCandidates) => groupedCandidates.zipWithIndex.map { case (candidate, index) => candidate.candidate.id -> candidateRescoringFactor(query, candidate, index) } case _ => Map.empty } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ControlAiRescorer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ControlAiEmbeddingSimilarityThresholdParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ControlAiShowLessScaleFactorParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ControlAiShowMoreScaleFactorParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableControlAiParam import com.twitter.home_mixer.product.scored_tweets.util.ControlAiUtil import com.twitter.product_mixer.component_library.feature_hydrator.query.control_ai.ControlAiTopicEmbeddingMapFeature import com.twitter.product_mixer.component_library.feature_hydrator.query.control_ai.UserControlAiFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.control_ai.control.{thriftscala => ci} sealed trait ControlAiRescorer extends RescoringFactorProvider object ControlAiRescorer { private def buildControlAiRescorer( actionType: ci.ActionType, factorParam: FSBoundedParam[Double] ): ControlAiRescorer = { new ControlAiRescorer { override def selector( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate] ): Boolean = { if (query.params(EnableControlAiParam)) { val actions = query.features .flatMap(_.getOrElse(UserControlAiFeature, None)) .map(_.actions).getOrElse(Seq.empty).filter(_.actionType == actionType) actions.exists( ControlAiUtil.conditionMatch( _, candidate, query.features.map(_.get(ControlAiTopicEmbeddingMapFeature)).getOrElse(Map.empty), threshold = query.params(ControlAiEmbeddingSimilarityThresholdParam) ) ) } else false } override def factor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate] ): Double = query.params(factorParam) } } val allRescorers = Seq( buildControlAiRescorer(ci.ActionType.More, ControlAiShowMoreScaleFactorParam), buildControlAiRescorer(ci.ActionType.Less, ControlAiShowLessScaleFactorParam) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/DeepRetrievalListwiseRescoringProvider.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.DeepRetrievalMaxCountParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableDeepRetrievalMaxCountParam import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery object DeepRetrievalListwiseRescoringProvider extends ListwiseRescoringProvider[CandidateWithFeatures[TweetCandidate], hmt.ServedType] { override def groupByKey( candidate: CandidateWithFeatures[TweetCandidate] ): Option[hmt.ServedType] = Some(candidate.features.get(ServedTypeFeature)) override def candidateRescoringFactor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate], index: Int ): Double = { if (query.params(EnableDeepRetrievalMaxCountParam)) { val servedType = candidate.features.get(ServedTypeFeature) val maxCount = query.params(DeepRetrievalMaxCountParam) if (servedType == hmt.ServedType.ForYouContentExplorationDeepRetrievalI2i && index >= maxCount) 0.0001 else 1.0 } else 1.0 } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Map[Long, Double] = { candidates .groupBy(groupByKey) .flatMap { case (_, groupedCandidates) => groupedCandidates.zipWithIndex.map { case (candidate, index) => candidate.candidate.id -> candidateRescoringFactor(query, candidate, index) } case _ => Map.empty } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/DiversityDiscountProvider.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures trait DiversityDiscountProvider { /** * Fetch the ID of the entity to diversify */ def entityId(candidate: CandidateWithFeatures[TweetCandidate]): Option[Long] /** * Compute discount factor for each candidate based on position (zero-based) * relative to other candidates associated with the same entity */ def discount(position: Int): Double /** * Return candidate IDs sorted by score in descending order */ def sort(candidates: Seq[CandidateWithFeatures[TweetCandidate]]): Seq[Long] = candidates .map { candidate => (candidate.candidate.id, candidate.features.getOrElse(ScoreFeature, None).getOrElse(0.0)) } .sortBy(_._2)(Ordering.Double.reverse) .map(_._1) /** * Group by the specified entity ID (e.g. authors, likers, followers) * Sort each group by score in descending order * Determine the discount factor based on the position of each candidate */ def apply( candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Map[Long, Double] = candidates .groupBy(entityId) .flatMap { case (entityIdOpt, entityCandidates) => val sortedCandidateIds = sort(entityCandidates) if (entityIdOpt.isDefined) { sortedCandidateIds.zipWithIndex.map { case (candidateId, index) => candidateId -> discount(index) } } else sortedCandidateIds.map(_ -> 1.0) } } object AuthorDiversityDiscountProvider extends DiversityDiscountProvider { private val Decay = 0.5 private val Floor = 0.25 override def entityId(candidate: CandidateWithFeatures[TweetCandidate]): Option[Long] = candidate.features.getOrElse(AuthorIdFeature, None) // Provides an exponential decay based discount by position (with a floor) override def discount(position: Int): Double = (1 - Floor) * Math.pow(Decay, position) + Floor } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/EvergreenDeepRetrievalCrossBorderListwiseRescoringProvider.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EvergreenDeepRetrievalCrossBorderMaxCountParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableEvergreenDeepRetrievalCrossBorderMaxCountParam import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery object EvergreenDeepRetrievalCrossBorderListwiseRescoringProvider extends ListwiseRescoringProvider[CandidateWithFeatures[TweetCandidate], hmt.ServedType] { override def groupByKey( candidate: CandidateWithFeatures[TweetCandidate] ): Option[hmt.ServedType] = Some(candidate.features.get(ServedTypeFeature)) override def candidateRescoringFactor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate], index: Int ): Double = { if (query.params(EnableEvergreenDeepRetrievalCrossBorderMaxCountParam)) { val servedType = candidate.features.get(ServedTypeFeature) val maxCount = query.params(EvergreenDeepRetrievalCrossBorderMaxCountParam) if (servedType == hmt.ServedType.ForYouEvergreenDeepRetrievalCrossBorderHome && index >= maxCount) 0.0001 else 1.0 } else 1.0 } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Map[Long, Double] = { candidates .groupBy(groupByKey) .flatMap { case (_, groupedCandidates) => groupedCandidates.zipWithIndex.map { case (candidate, index) => candidate.candidate.id -> candidateRescoringFactor(query, candidate, index) } case _ => Map.empty } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/EvergreenDeepRetrievalListwiseRescoringProvider.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EvergreenDeepRetrievalMaxCountParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableEvergreenDeepRetrievalMaxCountParam import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery object EvergreenDeepRetrievalListwiseRescoringProvider extends ListwiseRescoringProvider[CandidateWithFeatures[TweetCandidate], hmt.ServedType] { override def groupByKey( candidate: CandidateWithFeatures[TweetCandidate] ): Option[hmt.ServedType] = Some(candidate.features.get(ServedTypeFeature)) override def candidateRescoringFactor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate], index: Int ): Double = { if (query.params(EnableEvergreenDeepRetrievalMaxCountParam)) { val servedType = candidate.features.get(ServedTypeFeature) val maxCount = query.params(EvergreenDeepRetrievalMaxCountParam) if (servedType == hmt.ServedType.ForYouEvergreenDeepRetrievalHome && index >= maxCount) 0.0001 else 1.0 } else 1.0 } override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Map[Long, Double] = { candidates .groupBy(groupByKey) .flatMap { case (_, groupedCandidates) => groupedCandidates.zipWithIndex.map { case (candidate, index) => candidate.candidate.id -> candidateRescoringFactor(query, candidate, index) } case _ => Map.empty } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/GrokSlopScoreRescorer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.finagle.stats.DefaultStatsReceiver import com.twitter.home_mixer.model.HomeFeatures.GrokSlopScoreFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.GrokSlopScoreDecayValueParam import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery object GrokSlopScoreRescorer { private val treatmentValue = 3L private val statsReceiver = DefaultStatsReceiver.scope("GrokSlopScoreRescorer") private val numRescoredCandidatesCounter = statsReceiver.counter("rescored") private val totalCandidatesCounter = statsReceiver.counter("total") private def onlyIf(query: PipelineQuery): Boolean = { query.params(GrokSlopScoreDecayValueParam) < 1.0 } def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Map[Long, Double] = { if (!onlyIf(query)) { return candidates.map { candidate => candidate.candidate.id -> 1.0 }.toMap } val decayValue = query.params(GrokSlopScoreDecayValueParam) val rescoredCandidates = candidates.map { candidate => val featureValue = candidate.features.getOrElse(GrokSlopScoreFeature, None) val rescoreFactor = if (featureValue.contains(treatmentValue)) decayValue else 1.0 candidate.candidate.id -> rescoreFactor } numRescoredCandidatesCounter.incr(rescoredCandidates.count(_._2 != 1.0)) totalCandidatesCounter.incr(candidates.size) rescoredCandidates.toMap } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/HeuristicScorer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.home_mixer.model.HomeFeatures.PhoenixScoreFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnablePhoenixScorerParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.EnableNoNegHeuristicParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnablePhoenixScoreParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.MtlNormalization import com.twitter.home_mixer.util.RerankerUtil.Epsilon import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.timelines.util.MtlNormalizer object HeuristicScorer extends Scorer[PipelineQuery, TweetCandidate] { override val identifier: ScorerIdentifier = ScorerIdentifier("Heuristic") override val features: Set[Feature[_, _]] = Set(ScoreFeature) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { val noNegHeuristic = query.params(EnableNoNegHeuristicParam) val rescorers = Seq( RescoreOutOfNetwork, RescoreReplies, RescoreMTLNormalization( MtlNormalizer( alpha = query.params(MtlNormalization.AlphaParam) / 100.0, beta = query.params(MtlNormalization.BetaParam), gamma = query.params(MtlNormalization.GammaParam) ) ), RescoreListwise(ContentExplorationListwiseRescoringProvider(query, candidates)), RescoreListwise(DeepRetrievalListwiseRescoringProvider(query, candidates)), RescoreListwise(EvergreenDeepRetrievalListwiseRescoringProvider(query, candidates)), RescoreListwise( EvergreenDeepRetrievalCrossBorderListwiseRescoringProvider(query, candidates) ), RescoreListwise(AuthorBasedListwiseRescoringProvider(query, candidates)), RescoreListwise(ImpressedAuthorDecayRescoringProvider(query, candidates)), RescoreListwise(ImpressedMediaClusterBasedListwiseRescoringProvider(query, candidates)), RescoreListwise(ImpressedImageClusterBasedListwiseRescoringProvider(query, candidates)), RescoreListwise(CandidateSourceDiversityListwiseRescoringProvider(query, candidates)), RescoreListwise(GrokSlopScoreRescorer(query, candidates)), RescoreFeedbackFatigue(query), RescoreListwise(MultimodalEmbeddingRescorer(query, candidates)), RescoreLiveContent ) ++ ControlAiRescorer.allRescorers val usePhoenix = query.params(EnablePhoenixScorerParam) && query.params(EnablePhoenixScoreParam) val updatedScores = candidates.map { candidate => val scoreOpt = if (usePhoenix) candidate.features.getOrElse(PhoenixScoreFeature, None) else candidate.features.getOrElse(ScoreFeature, None) val scaleFactor = rescorers.map(_(query, candidate)).product val updatedScore = scoreOpt.map { score => if (score < Epsilon && noNegHeuristic) score else score * scaleFactor } FeatureMap(ScoreFeature, updatedScore) } Stitch.value(updatedScores) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ImpressedAuthorDecayRescoringProvider.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.ServedAuthorIdsFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableImpressionBasedAuthorDecay import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweets import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery object ImpressedAuthorDecayRescoringProvider { private def calculateAuthorImpressionFrequencies(query: PipelineQuery): Map[Long, Int] = { val impressedTweetIds = query.features.map(_.getOrElse(ImpressedTweets, Seq.empty)).getOrElse(Seq.empty).toSet val servedAuthorMap = query.features.map(_.get(ServedAuthorIdsFeature)).getOrElse(Map.empty) servedAuthorMap .map { case (authorId, tweetIds) => val impressedCount = tweetIds.count(impressedTweetIds.contains) authorId -> impressedCount } .filter(_._2 > 0) // Only include authors with at least one impressed tweet } private def groupByKey(candidate: CandidateWithFeatures[TweetCandidate]): Option[Long] = candidate.features.getOrElse(AuthorIdFeature, None) private def onlyIf(query: PipelineQuery): Boolean = query.params(EnableImpressionBasedAuthorDecay) def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Map[Long, Double] = { if (onlyIf(query)) { val outNetworkDecayFactor = query.params(ScoredTweetsParam.AuthorDiversityOutNetworkDecayFactor) val outNetworkFloor = query.params(ScoredTweetsParam.AuthorDiversityOutNetworkFloor) val inNetworkDecayFactor = query.params(ScoredTweetsParam.AuthorDiversityInNetworkDecayFactor) val inNetworkFloor = query.params(ScoredTweetsParam.AuthorDiversityInNetworkFloor) val authorFreq = calculateAuthorImpressionFrequencies(query) candidates .groupBy(groupByKey) .flatMap { case (Some(authorId), groupedCandidates) => val sortedCandidates = groupedCandidates .sortBy(_.features.getOrElse(ScoreFeature, None).getOrElse(0.0))( Ordering.Double.reverse) sortedCandidates.zipWithIndex.map { case (candidate, index) => candidate.candidate.id -> { val effectiveIndex = index + authorFreq.getOrElse(authorId, 0) val isInNetworkCandidate = candidate.features.getOrElse(InNetworkFeature, true) val decayFactor = if (isInNetworkCandidate) inNetworkDecayFactor else outNetworkDecayFactor val floor = if (isInNetworkCandidate) inNetworkFloor else outNetworkFloor authorDiversityBasedRescorer(effectiveIndex, decayFactor, floor) } } case _ => Map.empty } } else Map.empty } /** * Re-scoring multiplier to apply to multiple tweets from the same author. * Provides an exponential decay based discount by position (with a floor). */ private def authorDiversityBasedRescorer( index: Int, decayFactor: Double, floor: Double ): Double = (1 - floor) * Math.pow(decayFactor, index) + floor } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ImpressedImageClusterBasedListwiseRescoringProvider.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.finagle.stats.DefaultStatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.ImpressedImageClusterIds import com.twitter.home_mixer.model.HomeFeatures.ClipImageClusterIdsFeature import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ImpressedImageClusterBasedRescoringParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableImageClusterDecayParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableImageClusterFeatureHydrationParam import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery object ImpressedImageClusterBasedListwiseRescoringProvider { /** * Impressed Image cluster-Id based * Rescore the list of candidates that have previously impressed imageCluster Ids. */ private val impressedImageStats = DefaultStatsReceiver.scope("ImpressedImageClusterBasedListwiseRescoringProvider") private val percentRescoredCandidatesStat = impressedImageStats.stat("percent_rescored_candidates_x10") private def impressedImageClusterIds(query: PipelineQuery) = query.features.map(_.getOrElse(ImpressedImageClusterIds, Seq[Long]())).getOrElse(Seq[Long]()) private def imageClusterId( candidate: CandidateWithFeatures[TweetCandidate] ) = candidate.features.getOrElse(ClipImageClusterIdsFeature, Map[Long, Long]()).values.toSeq private def rescoringFactor( clusterIds: Seq[Long], impressedImageClusterIdsToCountMap: Map[Long, Int], decayFactor: Double ): Double = { val decayFactors = clusterIds.map { clusterId => impressedImageClusterIdsToCountMap.get(clusterId) match { case Some(count) => math.pow(1.0 - decayFactor, count) case None => 1.0 } } decayFactors.product } private def onlyIf(query: PipelineQuery): Boolean = { (query.params(ImpressedImageClusterBasedRescoringParam) > 0.0) && query.params(EnableImageClusterDecayParam) && query.params(EnableImageClusterFeatureHydrationParam) } def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Map[Long, Double] = { if (!onlyIf(query)) { return candidates.map { candidate => candidate.candidate.id -> 1.0 }.toMap } val impressedImageClusterIdsToCountMap = impressedImageClusterIds(query) .groupBy(identity) .mapValues(_.size) val decayFactor = query.params(ImpressedImageClusterBasedRescoringParam) val rescoringFactors = candidates.map { candidate => val imageClusterIds = imageClusterId(candidate) val rescoreFactor = rescoringFactor(imageClusterIds, impressedImageClusterIdsToCountMap, decayFactor) candidate.candidate.id -> rescoreFactor }.toMap // Update Rescored candidates stats by number of candidates with rescoreFactor != 1 percentRescoredCandidatesStat.add( (rescoringFactors.count(_._2 != 1.0) * 1000.0 / (candidates.size + 0.01)).toFloat) rescoringFactors } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ImpressedMediaClusterBasedListwiseRescoringProvider.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.finagle.stats.DefaultStatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.ImpressedMediaClusterIds import com.twitter.home_mixer.model.HomeFeatures.TweetMediaClusterIdsFeature import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ImpressedMediaClusterBasedRescoringParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableMediaClusterDecayParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableMediaClusterFeatureHydrationParam import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery object ImpressedMediaClusterBasedListwiseRescoringProvider { /** * Impressed Media cluster-Id based * Rescore the list of candidates that have previously impressed mediaCluster Ids. */ private val impressedMediaStats = DefaultStatsReceiver.scope("ImpressedMediaClusterBasedListwiseRescoringProvider") private val percentRescoredCandidatesStat = impressedMediaStats.stat("percent_rescored_candidates_x10") private def impressedMediaClusterIds(query: PipelineQuery) = query.features.map(_.getOrElse(ImpressedMediaClusterIds, Seq[Long]())).getOrElse(Seq[Long]()) private def mediaClusterId(candidate: CandidateWithFeatures[TweetCandidate]) = candidate.features.getOrElse(TweetMediaClusterIdsFeature, Map[Long, Long]()).values.toSeq private def rescoringFactor( clusterIds: Seq[Long], impressedMediaClusterIdsToCountMap: Map[Long, Int], decayFactor: Double ): Double = { val decayFactors = clusterIds.map { clusterId => impressedMediaClusterIdsToCountMap.get(clusterId) match { case Some(count) => math.pow(1.0 - decayFactor, count) case None => 1.0 } } decayFactors.product } private def onlyIf(query: PipelineQuery): Boolean = { (query.params(ImpressedMediaClusterBasedRescoringParam) > 0.0) && query.params(EnableMediaClusterDecayParam) && query.params(EnableMediaClusterFeatureHydrationParam) } def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Map[Long, Double] = { if (!onlyIf(query)) { return candidates.map { candidate => candidate.candidate.id -> 1.0 }.toMap } val impressedMediaClusterIdsToCountMap = impressedMediaClusterIds(query) .groupBy(identity) .mapValues(_.size) val decayFactor = query.params(ImpressedMediaClusterBasedRescoringParam) val rescoringFactors = candidates.map { candidate => val mediaClusterIds = mediaClusterId(candidate) val rescoreFactor = rescoringFactor(mediaClusterIds, impressedMediaClusterIdsToCountMap, decayFactor) candidate.candidate.id -> rescoreFactor }.toMap // Update Rescored candidates stats by number of candidates with rescoreFactor != 1 percentRescoredCandidatesStat.add( (rescoringFactors.count(_._2 != 1.0) * 1000.0 / (candidates.size + 0.01)).toFloat) rescoringFactors } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/ListwiseRescoringProvider.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery /** * Defines a listwise rescoring provider for use of rescoring a candidate dependent on the other candidates that * co-exist with it. * * Requires: * 1) groupByKey: How to define the set of candidates that are dependent on each other for scoring. * 2) candidateRescoringFactor: Compute the final rescoring factor for each candidate in the group. * * This rescorer will sort the grouped candidates by their current score, and then apply the rescoring factors to * each candidate in their group. */ trait ListwiseRescoringProvider[C <: CandidateWithFeatures[TweetCandidate], K] { /** * Fetch the key used to create groups of candidates */ def groupByKey(candidate: C): Option[K] /** * Compute the factor for each candidate based on position (zero-based) * relative to other candidates associated with the same key */ def candidateRescoringFactor(query: PipelineQuery, candidate: C, index: Int): Double /** * Group by the specified key (e.g. authors, likers, followers) * Sort each group by score in descending order * Determine the rescoring factor based on the position of each candidate */ def apply( query: PipelineQuery, candidates: Seq[C] ): Map[Long, Double] = candidates .groupBy(groupByKey) .flatMap { case (Some(_), groupedCandidates) => val sortedCandidates = groupedCandidates .sortBy(_.features.getOrElse(ScoreFeature, None).getOrElse(0.0))(Ordering.Double.reverse) sortedCandidates.zipWithIndex.map { case (candidate, index) => candidate.candidate.id -> candidateRescoringFactor(query, candidate, index) } case _ => Map.empty } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/LowSignalScorer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines import com.twitter.product_mixer.core.model.common.presentation.CandidateSourcePosition import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import scala.collection.immutable.ListSet /** * Assign scores based on candidate source positions, blending candidates from different sources */ object LowSignalScorer extends Scorer[PipelineQuery, TweetCandidate] { override val identifier: ScorerIdentifier = ScorerIdentifier("LowSignal") override val features: Set[Feature[_, _]] = Set(ScoreFeature) override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { val candidatesByPipeline = candidates.groupBy { _.features.getOrElse(CandidatePipelines, ListSet.empty[CandidatePipelineIdentifier]).head } val candidateScoreMap = candidatesByPipeline .map { case (_, pipelineCandidates) => val sortedCandidates = pipelineCandidates.sortBy(_.features.get(CandidateSourcePosition)) deduplicateAuthors(sortedCandidates).zipWithIndex.map { case (candidate, index) => candidate.candidate.id -> index.toDouble } }.toSeq.flatten.toMap val maxScore = candidates.size.toDouble val updatedScores = candidates.map { candidate => val score = maxScore - candidateScoreMap.getOrElse(candidate.candidate.id, maxScore) FeatureMap(ScoreFeature, Some(score)) } Stitch.value(updatedScores) } def deduplicateAuthors( candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Seq[CandidateWithFeatures[TweetCandidate]] = { val seenAuthors = scala.collection.mutable.Set[Long]() candidates.collect { case c if seenAuthors.add(c.features.getOrElse(AuthorIdFeature, None).getOrElse(0L)) => c } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/MultimodalEmbeddingRescorer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.finagle.stats.DefaultStatsReceiver import com.twitter.home_mixer.model.HomeFeatures.MultiModalEmbeddingsFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.MultiModalEmbeddingRescorerGammaParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.MultiModalEmbeddingRescorerMinScoreParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.MultiModalEmbeddingRescorerNumCandidatesParam import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery import scala.collection.mutable.ArrayBuffer object MultimodalEmbeddingRescorer { private val statsReceiver = DefaultStatsReceiver.scope("MultimodalEmbeddingRescorer") private val rescoredCandidatesStat = statsReceiver.stat("rescored_candidates") private val rescoreFactorx100Stat = statsReceiver.stat("rescore_factor_x100") private def onlyIf(query: PipelineQuery): Boolean = { (query.params(MultiModalEmbeddingRescorerGammaParam) > 0.0) && (query.params(MultiModalEmbeddingRescorerMinScoreParam) < 1.0) } private def dot(a: Array[Double], b: Array[Double]): Double = { a.zip(b).map { case (x, y) => x * y }.sum } private def getSimilarityScoreFactors( candidates: Seq[CandidateWithFeatures[TweetCandidate]], gamma: Double, minScore: Double ): Map[Long, Double] = { val embsBuilder = ArrayBuffer.empty[Array[Double]] val factorsBuilder = Map.newBuilder[Long, Double] candidates.foreach { candidate => val id = candidate.candidate.id candidate.features.getOrElse(MultiModalEmbeddingsFeature, None) match { case Some(embSeq: Seq[Double]) => val emb = embSeq.toArray var similarCount = 0 var i = 0 while (i < embsBuilder.length) { if (dot(embsBuilder(i), emb) > minScore) similarCount += 1 i += 1 } val factor = 1.0 / (1.0 + gamma * similarCount) factorsBuilder += (id -> factor) embsBuilder += emb case _ => } } factorsBuilder.result() } def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Map[Long, Double] = { if (onlyIf(query)) { // Get the top 100 candidates by score val sortedCandidates = candidates.sortBy(_.features.getOrElse(ScoreFeature, None).getOrElse(0.0)) // Get top 100 and rest of candidates val numCandidatesToScore = query.params(MultiModalEmbeddingRescorerNumCandidatesParam) val topCandidates = sortedCandidates.take(numCandidatesToScore) val restCandidates = sortedCandidates.drop(numCandidatesToScore) val candidateToFactorMap = getSimilarityScoreFactors( topCandidates, query.params(MultiModalEmbeddingRescorerGammaParam), query.params(MultiModalEmbeddingRescorerMinScoreParam) ) val rescoredCandidates = candidateToFactorMap.filter { case (_, factor) => factor != 1.0 } rescoredCandidatesStat.add(rescoredCandidates.size.toFloat) rescoreFactorx100Stat.add(rescoredCandidates.values.map(_.toFloat).sum * 100) // For the rest of the candidates, set factor to 1.0 restCandidates.map(candidate => candidate.candidate.id -> 1.0).toMap ++ candidateToFactorMap } else candidates.map(candidate => candidate.candidate.id -> 1.0).toMap } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/NaviModelScorer.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.finagle.stats.Stat import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.scorer.PredictedScoreFeature.PredictedScoreFeatures import com.twitter.ml.api.DataRecord import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.datarecord.AllFeatures import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordExtractor import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.stitch.Stitch import com.twitter.timelines.clients.predictionservice.PredictionGRPCService import com.twitter.timelines.clients.predictionservice.PredictionServiceGRPCClient import com.twitter.util.Future import com.twitter.util.Return import javax.inject.Inject import javax.inject.Singleton object CommonFeaturesDataRecordFeature extends DataRecordInAFeature[PipelineQuery] with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } object CandidateFeaturesDataRecordFeature extends DataRecordInAFeature[TweetCandidate] with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { override def defaultValue: DataRecord = new DataRecord() } @Singleton case class NaviModelScorer @Inject() ( predictionGRPCService: PredictionGRPCService, statsReceiver: StatsReceiver) extends Scorer[ScoredTweetsQuery, TweetCandidate] { override val identifier: ScorerIdentifier = ScorerIdentifier("NaviModel") override val features: Set[Feature[_, _]] = Set( CommonFeaturesDataRecordFeature, CandidateFeaturesDataRecordFeature, WeightedModelScoreFeature, ScoreFeature ) ++ PredictedScoreFeatures.asInstanceOf[Set[Feature[_, _]]] private val queryDataRecordAdapter = new DataRecordConverter(AllFeatures()) private val candidatesDataRecordAdapter = new DataRecordConverter(AllFeatures()) private val resultDataRecordExtractor = new DataRecordExtractor(PredictedScoreFeatures) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val failuresStat = scopedStatsReceiver.stat("failures") private val responsesStat = scopedStatsReceiver.stat("responses") private val invalidResponsesCounter = scopedStatsReceiver.counter("invalidResponses") private val candidatesDataRecordAdapterLatencyStat = scopedStatsReceiver.scope("candidatesDataRecordAdapter").stat("latency_ms") private val StatsReadabilityMultiplier = 1000 private val Epsilon = 0.001 private val PredictedScoreStatName = f"predictedScore${StatsReadabilityMultiplier}x" private val MissingScoreStatName = "missingScore" private val scoreStat = scopedStatsReceiver.stat(f"score${StatsReadabilityMultiplier}x") private val RequestBatchSize = 64 private val DataRecordConstructionParallelism = 32 private val ModelId = "Home" private val modelClient = new PredictionServiceGRPCClient( service = predictionGRPCService, statsReceiver = statsReceiver, requestBatchSize = RequestBatchSize, useCompact = false ) override def apply( query: ScoredTweetsQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { val commonRecord = query.features.map(queryDataRecordAdapter.toDataRecord) val candidateRecords: Future[Seq[DataRecord]] = Stat.time(candidatesDataRecordAdapterLatencyStat) { OffloadFuturePools.parallelize[FeatureMap, DataRecord]( inputSeq = candidates.map(_.features), transformer = candidatesDataRecordAdapter.toDataRecord(_), parallelism = DataRecordConstructionParallelism, default = new DataRecord ) } val scoreFeatureMaps = candidateRecords.flatMap { records => val predictionResponses = modelClient.getPredictions(records, commonRecord, modelId = Some(ModelId)) predictionResponses.map { responses => failuresStat.add(responses.count(_.isThrow)) responsesStat.add(responses.size) if (responses.size == candidates.size) { val predictedScoreFeatureMaps = responses.map { case Return(dataRecord) => resultDataRecordExtractor.fromDataRecord(dataRecord) case _ => resultDataRecordExtractor.fromDataRecord(new DataRecord()) } // Add Data Record to candidate Feature Map for logging in later stages predictedScoreFeatureMaps.zip(records).map { case (predictedScoreFeatureMap, candidateRecord) => val weightedModelScore = computeWeightedModelScore(query, predictedScoreFeatureMap) scoreStat.add((weightedModelScore * StatsReadabilityMultiplier).toFloat) predictedScoreFeatureMap + (CandidateFeaturesDataRecordFeature, candidateRecord) + (CommonFeaturesDataRecordFeature, commonRecord.getOrElse(new DataRecord())) + (ScoreFeature, Some(weightedModelScore)) + (WeightedModelScoreFeature, Some(weightedModelScore)) } } else { invalidResponsesCounter.incr() throw PipelineFailure(IllegalStateFailure, "Result size mismatched candidates size") } } } Stitch.callFuture(scoreFeatureMaps) } /** * Compute the weighted sum of predicted scores of all engagements * Convert negative score to positive, if needed */ private def computeWeightedModelScore( query: PipelineQuery, features: FeatureMap ): Double = { val weightedScoreAndModelWeightSeq = PredictedScoreFeatures.toSeq.map { predictedScoreFeature => val predictedScoreOpt = predictedScoreFeature.extractScore(features) predictedScoreOpt match { case Some(predictedScore) => scopedStatsReceiver .stat(predictedScoreFeature.statName, PredictedScoreStatName) .add((predictedScore * StatsReadabilityMultiplier).toFloat) case None => scopedStatsReceiver.counter(predictedScoreFeature.statName, MissingScoreStatName).incr() } val weight = query.params(predictedScoreFeature.modelWeightParam) val weightedScore = predictedScoreOpt.getOrElse(0.0) * weight (weightedScore, weight) } val (weightedScores, modelWeights) = weightedScoreAndModelWeightSeq.unzip val combinedScoreSum = weightedScores.sum val positiveModelWeightsSum = modelWeights.filter(_ > 0.0).sum val negativeModelWeightsSum = modelWeights.filter(_ < 0).sum.abs val modelWeightsSum = positiveModelWeightsSum + negativeModelWeightsSum val weightedScoresSum = if (modelWeightsSum == 0) combinedScoreSum.max(0.0) else if (combinedScoreSum < 0) (combinedScoreSum + negativeModelWeightsSum) / modelWeightsSum * Epsilon else combinedScoreSum + Epsilon weightedScoresSum } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/PredictedScoreFeature.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.dal.personal_data.{thriftjava => pd} import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.Scoring.ModelWeights import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.prediction.features.recap.RecapFeatures sealed trait PredictedScoreFeature extends DataRecordOptionalFeature[TweetCandidate, Double] with DoubleDataRecordCompatible { override val personalDataTypes: Set[pd.PersonalDataType] = Set.empty def statName: String def modelWeightParam: FSBoundedParam[Double] def extractScore: FeatureMap => Option[Double] = _.getOrElse(this, None) } object PredictedFavoriteScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_FAVORITED.getFeatureName override val statName = "fav" override val modelWeightParam = ModelWeights.FavParam } object PredictedReplyScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_REPLIED.getFeatureName override val statName = "reply" override val modelWeightParam = ModelWeights.ReplyParam } object PredictedRetweetScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_RETWEETED.getFeatureName override val statName = "retweet" override val modelWeightParam = ModelWeights.RetweetParam } object PredictedReplyEngagedByAuthorScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_REPLIED_REPLY_ENGAGED_BY_AUTHOR.getFeatureName override val statName = "reply_engaged_by_author" override val modelWeightParam = ModelWeights.ReplyEngagedByAuthorParam } object PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_GOOD_CLICKED_V1.getFeatureName override val statName = "good_click_convo_desc_favorited_or_replied" override val modelWeightParam = ModelWeights.GoodClickParam override def extractScore: FeatureMap => Option[Double] = { featureMap => val goodClickV1Opt = featureMap.getOrElse(this, None) val goodClickV2Opt = featureMap.getOrElse(PredictedGoodClickConvoDescUamGt2ScoreFeature, None) (goodClickV1Opt, goodClickV2Opt) match { case (Some(v1Score), Some(v2Score)) => Some(Math.max(v1Score, v2Score)) case _ => goodClickV1Opt.orElse(goodClickV2Opt) } } } object PredictedGoodClickConvoDescUamGt2ScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_GOOD_CLICKED_V2.getFeatureName override val statName = "good_click_convo_desc_uam_gt_2" override val modelWeightParam = ModelWeights.GoodClickV2Param } object PredictedGoodProfileClickScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_PROFILE_CLICKED_AND_PROFILE_ENGAGED.getFeatureName override val statName = "good_profile_click" override val modelWeightParam = ModelWeights.GoodProfileClickParam } object PredictedVideoPlayback50ScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_VIDEO_PLAYBACK_50.getFeatureName override val statName = "video_playback_50" override val modelWeightParam = ModelWeights.VideoPlayback50Param } object PredictedTweetDetailDwellScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_TWEET_DETAIL_DWELLED_15_SEC.getFeatureName override val statName = "tweet_detail_dwell" override val modelWeightParam = ModelWeights.TweetDetailDwellParam } object PredictedProfileDwelledScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_PROFILE_DWELLED_20_SEC.getFeatureName override val statName = "profile_dwell" override val modelWeightParam = ModelWeights.ProfileDwelledParam } object PredictedBookmarkScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_BOOKMARKED.getFeatureName override val statName = "bookmark" override val modelWeightParam = ModelWeights.BookmarkParam } object PredictedShareScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_SHARED.getFeatureName override val statName = "share" override val modelWeightParam = ModelWeights.ShareParam } object PredictedShareMenuClickScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_SHARE_MENU_CLICKED.getFeatureName override val statName = "share_menu_click" override val modelWeightParam = ModelWeights.ShareMenuClickParam } // Negative Engagements object PredictedNegativeFeedbackV2ScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_NEGATIVE_FEEDBACK_V2.getFeatureName override val statName = "negative_feedback_v2" override val modelWeightParam = ModelWeights.NegativeFeedbackV2Param } object PredictedReportedScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_REPORT_TWEET_CLICKED.getFeatureName override val statName = "reported" override val modelWeightParam = ModelWeights.ReportParam } object PredictedStrongNegativeFeedbackScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_STRONG_NEGATIVE_FEEDBACK.getFeatureName override val statName = "strong_negative_feedback" override val modelWeightParam = ModelWeights.StrongNegativeFeedbackParam } object PredictedWeakNegativeFeedbackScoreFeature extends PredictedScoreFeature { override val featureName: String = RecapFeatures.PREDICTED_IS_WEAK_NEGATIVE_FEEDBACK.getFeatureName override val statName = "weak_negative_feedback" override val modelWeightParam = ModelWeights.WeakNegativeFeedbackParam } object PredictedScoreFeature { val PredictedScoreFeatures: Set[PredictedScoreFeature] = Set( PredictedFavoriteScoreFeature, PredictedReplyScoreFeature, PredictedRetweetScoreFeature, PredictedReplyEngagedByAuthorScoreFeature, PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, PredictedGoodClickConvoDescUamGt2ScoreFeature, PredictedGoodProfileClickScoreFeature, PredictedVideoPlayback50ScoreFeature, PredictedTweetDetailDwellScoreFeature, PredictedProfileDwelledScoreFeature, PredictedBookmarkScoreFeature, PredictedShareScoreFeature, PredictedShareMenuClickScoreFeature, // Negative Engagements PredictedNegativeFeedbackV2ScoreFeature, PredictedReportedScoreFeature, PredictedStrongNegativeFeedbackScoreFeature, PredictedWeakNegativeFeedbackScoreFeature, ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer/RescoringFactorProvider.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scorer import com.twitter.home_mixer.functional_component.feature_hydrator.BroadcastStateFeature import com.twitter.home_mixer.functional_component.feature_hydrator.SpaceStateFeature import com.twitter.home_mixer.functional_component.scorer.FeedbackFatigueScorer import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam._ import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.util.MtlNormalizer import com.twitter.timelineservice.{thriftscala => tls} import com.twitter.ubs.{thriftscala => ubs} trait RescoringFactorProvider { def selector(query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate]): Boolean def factor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate] ): Double def apply( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate], ): Double = if (selector(query, candidate)) factor(query, candidate) else 1.0 } case class RescoreListwise(listwiseRescoringMap: Map[Long, Double]) extends RescoringFactorProvider { override def selector( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate] ): Boolean = listwiseRescoringMap.contains(candidate.candidate.id) override def factor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate] ): Double = listwiseRescoringMap(candidate.candidate.id) } /** * Re-scoring multiplier to apply to out-of-network tweets */ object RescoreOutOfNetwork extends RescoringFactorProvider { def selector(query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate]): Boolean = !candidate.features.getOrElse(InNetworkFeature, false) def factor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate] ): Double = query.params(OutOfNetworkScaleFactorParam) } /** * Re-scoring multiplier to apply to reply candidates */ object RescoreReplies extends RescoringFactorProvider { def selector(query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate]): Boolean = candidate.features.getOrElse(InReplyToTweetIdFeature, None).isDefined def factor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate] ): Double = query.params(ReplyScaleFactorParam) } /** * Re-scoring multiplier to calibrate multi-tasks learning model prediction */ case class RescoreMTLNormalization(mtlNormalizer: MtlNormalizer) extends RescoringFactorProvider { def selector(query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate]): Boolean = query.params(MtlNormalization.EnableMtlNormalizationParam) def factor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate] ): Double = mtlNormalizer( attribute = candidate.features.getOrElse(AuthorFollowersFeature, None), retweet = candidate.features.getOrElse(SourceTweetIdFeature, None).isDefined, reply = candidate.features.getOrElse(InReplyToTweetIdFeature, None).isDefined ) } case class RescoreFeedbackFatigue(query: PipelineQuery) extends RescoringFactorProvider { def selector(query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate]): Boolean = true private val feedbackEntriesByEngagementType = query.features .getOrElse(FeatureMap.empty).getOrElse(FeedbackHistoryFeature, Seq.empty) .filter { entry => val timeSinceFeedback = query.queryTime.minus(entry.timestamp) timeSinceFeedback < FeedbackFatigueScorer.DurationForDiscounting && entry.feedbackType == tls.FeedbackType.SeeFewer }.groupBy(_.engagementType) private val authorsToDiscount = FeedbackFatigueScorer.getUserDiscounts( query.queryTime, feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Tweet, Seq.empty)) private val likersToDiscount = FeedbackFatigueScorer.getUserDiscounts( query.queryTime, feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Like, Seq.empty)) private val followersToDiscount = FeedbackFatigueScorer.getUserDiscounts( query.queryTime, feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Follow, Seq.empty)) private val retweetersToDiscount = FeedbackFatigueScorer.getUserDiscounts( query.queryTime, feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty)) def factor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate] ): Double = { FeedbackFatigueScorer.getScoreMultiplier( candidate, authorsToDiscount, likersToDiscount, followersToDiscount, retweetersToDiscount ) } } /** * Disabled in production */ object RescoreLiveContent extends RescoringFactorProvider { private val MinFollowers = 1000000 def selector(query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate]): Boolean = { ( candidate.features.getOrElse(SpaceStateFeature, None).contains(ubs.BroadcastState.Running) || candidate.features.getOrElse(BroadcastStateFeature, None).contains(ubs.BroadcastState.Running) ) && candidate.features.getOrElse(InNetworkFeature, false) && candidate.features.getOrElse(AuthorFollowersFeature, None).exists(_ > MinFollowers) } def factor( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate] ): Double = query.params(LiveContentScaleFactorParam) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer-features/thrift/src/main/thrift:thrift-scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/real_time_aggregates", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/embedding", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/real_time_aggregates", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/param_gated", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring", "servo/repo/src/main/scala", "src/scala/com/twitter/timelines/prediction/adapters/large_embeddings", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsHeuristicScoringPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scoring_pipeline import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsStaticCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.gate.DenyLowSignalUserGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.home_mixer.product.scored_tweets.scorer.HeuristicScorer import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.core.functional_component.common.AllExceptPipelines import com.twitter.product_mixer.core.functional_component.gate.BaseGate import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig import com.twitter.timelines.configapi.FSParam object ScoredTweetsHeuristicScoringPipelineConfig extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] { override val identifier: ScoringPipelineIdentifier = ScoringPipelineIdentifier("ScoredTweetsHeuristic") override val supportedClientParam: Option[FSParam[Boolean]] = Some(ScoredTweetsParam.EnableHeuristicScoringPipeline) override val gates: Seq[BaseGate[ScoredTweetsQuery]] = Seq(DenyLowSignalUserGate) private val allExcept = AllExceptPipelines( pipelinesToExclude = Set(ScoredTweetsStaticCandidatePipelineConfig.Identifier) ) override val selectors: Seq[Selector[ScoredTweetsQuery]] = Seq(InsertAppendResults(allExcept)) override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = Seq(HeuristicScorer) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsLowSignalScoringPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scoring_pipeline import com.twitter.home_mixer.product.scored_tweets.gate.AllowLowSignalUserGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.scorer.LowSignalScorer import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.core.functional_component.common.AllPipelines import com.twitter.product_mixer.core.functional_component.gate.BaseGate import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig object ScoredTweetsLowSignalScoringPipelineConfig extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] { override val identifier: ScoringPipelineIdentifier = ScoringPipelineIdentifier("ScoredTweetsLowSignal") override val gates: Seq[BaseGate[ScoredTweetsQuery]] = Seq(AllowLowSignalUserGate) override val selectors: Seq[Selector[ScoredTweetsQuery]] = Seq(InsertAppendResults(AllPipelines)) override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = Seq(LowSignalScorer) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsModelScoringPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scoring_pipeline import com.twitter.home_mixer.functional_component.feature_hydrator._ import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.TopicEdgeAggregateFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.TopicEdgeTruncatedAggregateFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.TweetContentEdgeAggregateFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.UserEngagerEdgeAggregateFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.UserEntityAggregateFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.MediaClusterIdFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.real_time_aggregates._ import com.twitter.home_mixer.functional_component.scorer.NaviModelScorer import com.twitter.home_mixer.functional_component.scorer.PhoenixScorer import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature import com.twitter.home_mixer.param.HomeGlobalParams.EnablePhoenixScorerParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableGeoduckAuthorLocationHydatorParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableSimclustersSparseTweetFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTransformerPostEmbeddingJointBlueFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTweetLanguageFeaturesParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinRebuildTweetFeaturesOnlineParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableTwhinTweetFeaturesOnlineParam import com.twitter.home_mixer.param.HomeGlobalParams.FeatureHydration.EnableViewCountFeaturesParam import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.CachedScoredTweetsCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsBackfillCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsDirectUtegCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsListsCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsStaticCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsTweetMixerCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird.ScoredTweetsCommunitiesCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird.ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.EarlybirdFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ReplyFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.SemanticCoreFeatureHydrator import com.twitter.home_mixer.product.scored_tweets.gate.DenyLowSignalUserGate import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableClipImagesClusterIdFeatureHydrationParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableMediaCompletionRateFeatureHydrationParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableMultiModalEmbeddingsFeatureHydratorParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.EnableTweetTextV8EmbeddingFeatureParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.QualityFactor import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.product_mixer.component_library.feature_hydrator.candidate.embedding.TweetTextV8EmbeddingFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedBulkCandidateFeatureHydrator import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.scorer.param_gated.ParamGatedScorer import com.twitter.product_mixer.component_library.selector.DropMaxCandidates import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates import com.twitter.product_mixer.core.functional_component.common.AllExceptPipelines import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator import com.twitter.product_mixer.core.functional_component.gate.BaseGate import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.product_mixer.core.pipeline.pipeline_failure.UnexpectedCandidateResult import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig import com.twitter.timelines.configapi.Param import javax.inject.Inject import javax.inject.Singleton @Singleton class ScoredTweetsModelScoringPipelineConfig @Inject() ( // Candidate sources scoredTweetsTweetMixerCandidatePipelineConfig: ScoredTweetsTweetMixerCandidatePipelineConfig, scoredTweetsListsCandidatePipelineConfig: ScoredTweetsListsCandidatePipelineConfig, scoredTweetsBackfillCandidatePipelineConfig: ScoredTweetsBackfillCandidatePipelineConfig, scoredTweetsEarlybirdInNetworkCandidatePipelineConfig: ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig, scoredTweetsCommunitiesCandidatePipelineConfig: ScoredTweetsCommunitiesCandidatePipelineConfig, scoredTweetsDirectUtegCandidatePipelineConfig: ScoredTweetsDirectUtegCandidatePipelineConfig, // Feature hydrators ancestorFeatureHydrator: AncestorFeatureHydrator, authorFeatureHydrator: AuthorFeatureHydrator, broadcastStateFeatureHydrator: BroadcastStateFeatureHydrator, earlybirdFeatureHydrator: EarlybirdFeatureHydrator, gizmoduckAuthorFeatureHydrator: GizmoduckAuthorFeatureHydrator, geoduckAuthorLocationHydrator: GeoduckAuthorLocationHydrator, graphTwoHopFeatureHydrator: GraphTwoHopFeatureHydrator, mediaClusterIdFeatureHydrator: MediaClusterIdFeatureHydrator, mediaCompletionRateFeatureHydrator: MediaCompletionRateFeatureHydrator, clipImagesClusterIdFeatureHydrator: ClipImageClusterIdFeatureHydrator, viralContentCreatorMetricsFeatureHydrator: ViralContentCreatorMetricsFeatureHydrator, multiModalEmbeddingsFeatureHydrator: MultiModalEmbeddingsFeatureHydrator, realGraphViewerAuthorFeatureHydrator: RealGraphViewerAuthorFeatureHydrator, realGraphViewerRelatedUsersFeatureHydrator: RealGraphViewerRelatedUsersFeatureHydrator, realTimeInteractionGraphEdgeFeatureHydrator: RealTimeInteractionGraphEdgeFeatureHydrator, replyFeatureHydrator: ReplyFeatureHydrator, sgsValidSocialContextFeatureHydrator: SGSValidSocialContextFeatureHydrator, simClustersEngagementSimilarityFeatureHydrator: SimClustersEngagementSimilarityFeatureHydrator, simClustersUserTweetScoresHydrator: SimClustersUserTweetScoresHydrator, spaceStateFeatureHydrator: SpaceStateFeatureHydrator, tspInferredTopicFeatureHydrator: TSPInferredTopicFeatureHydrator, tweetEntityServiceContentFeatureHydrator: TweetEntityServiceContentFeatureHydrator, tweetTextV8EmbeddingFeatureHydrator: TweetTextV8EmbeddingFeatureHydrator, twhinAuthorFollowFeatureHydrator: TwhinAuthorFollowFeatureHydrator, twhinTweetFeatureHydrator: TwhinTweetFeatureHydrator, twhinRebuildTweetFeatureHydrator: TwhinRebuildTweetFeatureHydrator, utegFeatureHydrator: UtegFeatureHydrator, slopAuthorFeatureHydrator: SlopAuthorFeatureHydrator, grokAnnotationsFeatureHydrator: GrokAnnotationsFeatureHydrator, // Real time aggregate feature hydrators engagementsReceivedByAuthorRealTimeAggregateFeatureHydrator: EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator, topicCountryEngagementRealTimeAggregateFeatureHydrator: TopicCountryEngagementRealTimeAggregateFeatureHydrator, topicEngagementRealTimeAggregateFeatureHydrator: TopicEngagementRealTimeAggregateFeatureHydrator, tweetCountryEngagementRealTimeAggregateFeatureHydrator: TweetCountryEngagementRealTimeAggregateFeatureHydrator, tweetEngagementRealTimeAggregateFeatureHydrator: TweetEngagementRealTimeAggregateFeatureHydrator, tweetLanguageFeatureHydrator: TweetLanguageFeatureHydrator, twitterListEngagementRealTimeAggregateFeatureHydrator: TwitterListEngagementRealTimeAggregateFeatureHydrator, userAuthorEngagementRealTimeAggregateFeatureHydrator: UserAuthorEngagementRealTimeAggregateFeatureHydrator, viewCountsFeatureHydrator: ViewCountsFeatureHydrator, // Large embeddings hydrators authorLargeEmbeddingsFeatureHydrator: AuthorLargeEmbeddingsFeatureHydrator, originalAuthorLargeEmbeddingsFeatureHydrator: OriginalAuthorLargeEmbeddingsFeatureHydrator, tweetLargeEmbeddingsFeatureHydrator: TweetLargeEmbeddingsFeatureHydrator, originalTweetLargeEmbeddingsFeatureHydrator: OriginalTweetLargeEmbeddingsFeatureHydrator, // Transformer embeddings hydrators transformerPostEmbeddingBlueFeatureHydrator: TransformerPostEmbeddingHomeBlueFeatureHydrator, transformerPostEmbeddingGreenFeatureHydrator: TransformerPostEmbeddingHomeGreenFeatureHydrator, transformerPostEmbeddingJointBlueFeatureHydrator: TransformerPostEmbeddingJointBlueFeatureHydrator, simClustersLogFavBasedTweetFeatureHydrator: SimClustersLogFavBasedTweetFeatureHydrator, // Scorers naviModelScorer: NaviModelScorer, phoenixScorer: PhoenixScorer) extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] { override val identifier: ScoringPipelineIdentifier = ScoringPipelineIdentifier("ScoredTweetsModel") private val nonCachedScoringPipelineScope = AllExceptPipelines( pipelinesToExclude = Set( CachedScoredTweetsCandidatePipelineConfig.Identifier, ScoredTweetsStaticCandidatePipelineConfig.Identifier ) ) override val gates: Seq[BaseGate[ScoredTweetsQuery]] = Seq( DenyLowSignalUserGate, NonEmptyCandidatesGate(nonCachedScoringPipelineScope) ) private val earlybirdScorePipelineScope = Set( scoredTweetsEarlybirdInNetworkCandidatePipelineConfig.identifier, scoredTweetsDirectUtegCandidatePipelineConfig.identifier ) private val earlybirdScoreOrdering: Ordering[CandidateWithDetails] = Ordering.by[CandidateWithDetails, Double] { case ItemCandidateWithDetails(_, _, features) => -features.getOrElse(EarlybirdScoreFeature, None).getOrElse(0.0) case _ => throw PipelineFailure(UnexpectedCandidateResult, "Invalid candidate type") } private def qualityFactorDropMaxCandidates( pipelineIdentifier: CandidatePipelineIdentifier, qualityFactorParam: Param[Int] ): DropMaxCandidates[ScoredTweetsQuery] = { new DropMaxCandidates( pipelineScope = SpecificPipelines(pipelineIdentifier), maxSelector = (query, _, _) => (query.getQualityFactorCurrentValue(identifier) * query.params(qualityFactorParam)).toInt ) } override val selectors: Seq[Selector[ScoredTweetsQuery]] = Seq( UpdateSortCandidates(SpecificPipelines(earlybirdScorePipelineScope), earlybirdScoreOrdering), UpdateSortCandidates( SpecificPipeline(scoredTweetsBackfillCandidatePipelineConfig.identifier), CandidatesUtil.reverseChronTweetsOrdering ), qualityFactorDropMaxCandidates( scoredTweetsTweetMixerCandidatePipelineConfig.identifier, QualityFactor.TweetMixerMaxTweetsToScoreParam ), qualityFactorDropMaxCandidates( scoredTweetsListsCandidatePipelineConfig.identifier, QualityFactor.ListsMaxTweetsToScoreParam ), qualityFactorDropMaxCandidates( scoredTweetsBackfillCandidatePipelineConfig.identifier, QualityFactor.BackfillMaxTweetsToScoreParam ), qualityFactorDropMaxCandidates( scoredTweetsEarlybirdInNetworkCandidatePipelineConfig.identifier, QualityFactor.InNetworkMaxTweetsToScoreParam ), qualityFactorDropMaxCandidates( scoredTweetsCommunitiesCandidatePipelineConfig.identifier, QualityFactor.CommunitiesMaxTweetsToScoreParam ), qualityFactorDropMaxCandidates( scoredTweetsDirectUtegCandidatePipelineConfig.identifier, QualityFactor.UtegMaxTweetsToScoreParam ), // Select candidates for Heavy Ranker Feature Hydration and Scoring InsertAppendResults(nonCachedScoringPipelineScope) ) override val preScoringFeatureHydrationPhase1: Seq[ BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] ] = Seq( DependentBulkCandidateFeatureHydrator(ancestorFeatureHydrator, Seq(replyFeatureHydrator)), authorFeatureHydrator, DependentBulkCandidateFeatureHydrator( earlybirdFeatureHydrator, Seq(spaceStateFeatureHydrator, broadcastStateFeatureHydrator, TweetTimeFeatureHydrator)), gizmoduckAuthorFeatureHydrator, ParamGatedBulkCandidateFeatureHydrator( EnableGeoduckAuthorLocationHydatorParam, geoduckAuthorLocationHydrator, ), graphTwoHopFeatureHydrator, InNetworkFeatureHydrator, realGraphViewerAuthorFeatureHydrator, realTimeInteractionGraphEdgeFeatureHydrator, simClustersEngagementSimilarityFeatureHydrator, simClustersUserTweetScoresHydrator, DependentBulkCandidateFeatureHydrator( tspInferredTopicFeatureHydrator, Seq( TopicEdgeAggregateFeatureHydrator, TopicEdgeTruncatedAggregateFeatureHydrator, topicCountryEngagementRealTimeAggregateFeatureHydrator, topicEngagementRealTimeAggregateFeatureHydrator ) ), TweetMetaDataFeatureHydrator, ParamGatedBulkCandidateFeatureHydrator( EnableTweetTextV8EmbeddingFeatureParam, tweetTextV8EmbeddingFeatureHydrator ), DependentBulkCandidateFeatureHydrator( tweetEntityServiceContentFeatureHydrator, Seq( SemanticCoreFeatureHydrator, TweetContentEdgeAggregateFeatureHydrator, mediaClusterIdFeatureHydrator)), twhinAuthorFollowFeatureHydrator, ParamGatedBulkCandidateFeatureHydrator( EnableTwhinTweetFeaturesOnlineParam, twhinTweetFeatureHydrator ), ParamGatedBulkCandidateFeatureHydrator( EnableTwhinRebuildTweetFeaturesOnlineParam, twhinRebuildTweetFeatureHydrator ), ParamGatedBulkCandidateFeatureHydrator( EnableViewCountFeaturesParam, viewCountsFeatureHydrator ), DependentBulkCandidateFeatureHydrator( utegFeatureHydrator, Seq( realGraphViewerRelatedUsersFeatureHydrator, sgsValidSocialContextFeatureHydrator, UserEngagerEdgeAggregateFeatureHydrator ) ), slopAuthorFeatureHydrator, // Real time aggregates engagementsReceivedByAuthorRealTimeAggregateFeatureHydrator, tweetCountryEngagementRealTimeAggregateFeatureHydrator, tweetEngagementRealTimeAggregateFeatureHydrator, ParamGatedBulkCandidateFeatureHydrator( EnableTweetLanguageFeaturesParam, tweetLanguageFeatureHydrator ), twitterListEngagementRealTimeAggregateFeatureHydrator, userAuthorEngagementRealTimeAggregateFeatureHydrator, // Offline aggregates UserEntityAggregateFeatureHydrator, viralContentCreatorMetricsFeatureHydrator, GrokGorkContentCreatorFeatureHydrator, // Large Embeddings authorLargeEmbeddingsFeatureHydrator, originalAuthorLargeEmbeddingsFeatureHydrator, tweetLargeEmbeddingsFeatureHydrator, originalTweetLargeEmbeddingsFeatureHydrator, // Transformers transformerPostEmbeddingBlueFeatureHydrator, transformerPostEmbeddingGreenFeatureHydrator, ParamGatedBulkCandidateFeatureHydrator( EnableTransformerPostEmbeddingJointBlueFeaturesParam, transformerPostEmbeddingJointBlueFeatureHydrator ), ParamGatedBulkCandidateFeatureHydrator( EnableSimclustersSparseTweetFeaturesParam, simClustersLogFavBasedTweetFeatureHydrator ), grokAnnotationsFeatureHydrator, ParamGatedBulkCandidateFeatureHydrator( EnableMediaCompletionRateFeatureHydrationParam, mediaCompletionRateFeatureHydrator, ), ParamGatedBulkCandidateFeatureHydrator( EnableClipImagesClusterIdFeatureHydrationParam, clipImagesClusterIdFeatureHydrator ), ParamGatedBulkCandidateFeatureHydrator( EnableMultiModalEmbeddingsFeatureHydratorParam, multiModalEmbeddingsFeatureHydrator ) ) override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = Seq( naviModelScorer, ParamGatedScorer(EnablePhoenixScorerParam, phoenixScorer) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline/ScoredTweetsRerankingScoringPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.scoring_pipeline import com.twitter.home_mixer.functional_component.scorer.PhoenixModelRerankingScorer import com.twitter.home_mixer.functional_component.scorer.WeighedModelRerankingScorer import com.twitter.home_mixer.param.HomeGlobalParams.EnablePhoenixScorerParam import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.scorer.param_gated.ParamGatedScorer import com.twitter.product_mixer.core.functional_component.gate.BaseGate import com.twitter.product_mixer.core.functional_component.scorer.Scorer import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig import javax.inject.Inject import javax.inject.Singleton @Singleton class ScoredTweetsRerankingScoringPipelineConfig @Inject() ( // rerankers weighedModelRerankingScorer: WeighedModelRerankingScorer, phoenixModelRerankingScorer: PhoenixModelRerankingScorer, // Base scoring pipeline config scoredTweetsModelScoringPipelineConfig: ScoredTweetsModelScoringPipelineConfig) extends ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate] { override val identifier: ScoringPipelineIdentifier = ScoringPipelineIdentifier("ScoredTweetsReranking") override val gates: Seq[BaseGate[ScoredTweetsQuery]] = scoredTweetsModelScoringPipelineConfig.gates override val selectors: Seq[Selector[ScoredTweetsQuery]] = scoredTweetsModelScoringPipelineConfig.selectors override val scorers: Seq[Scorer[ScoredTweetsQuery, TweetCandidate]] = Seq( weighedModelRerankingScorer, ParamGatedScorer(EnablePhoenixScorerParam, phoenixModelRerankingScorer) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/selector", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.selector import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.selector.SelectorResult import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery case class KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType( override val pipelineScope: CandidateScope) extends Selector[PipelineQuery] { override def apply( query: PipelineQuery, remainingCandidates: Seq[CandidateWithDetails], result: Seq[CandidateWithDetails] ): SelectorResult = { val (selectedCandidates, otherCandidates) = remainingCandidates.partition(candidate => pipelineScope.contains(candidate) && !candidate.features.getOrElse(InNetworkFeature, true)) val filteredCandidates = selectedCandidates .groupBy { candidate => ( candidate.features.getOrElse(AuthorIdFeature, None), candidate.features.getOrElse(SuggestTypeFeature, None) ) } .values.map(_.maxBy(_.features.getOrElse(ScoreFeature, None))) .toSeq val updatedCandidates = otherCandidates ++ filteredCandidates SelectorResult(remainingCandidates = updatedCandidates, result = result) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector/KeepTopKCandidatesPerCommunity.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.selector import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature import com.twitter.product_mixer.core.functional_component.common.CandidateScope import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.selector.SelectorResult import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery case class KeepTopKCandidatesPerCommunity(override val pipelineScope: CandidateScope) extends Selector[PipelineQuery] { private val MaxCandidatesPerCommunity = 1 private val MaxCommunityCandidates = 3 override def apply( query: PipelineQuery, remainingCandidates: Seq[CandidateWithDetails], result: Seq[CandidateWithDetails] ): SelectorResult = { val (selectedCandidates, otherCandidates) = remainingCandidates.partition { candidate => pipelineScope.contains(candidate) && candidate.features.getOrElse(CommunityIdFeature, None).isDefined } val filteredCandidates = selectedCandidates .groupBy { candidate => candidate.features.getOrElse(CommunityIdFeature, None) } .values.flatMap { _.sortBy(_.features.getOrElse(ScoreFeature, None)).reverse.take(MaxCandidatesPerCommunity) } .toSeq.sortBy(_.features.getOrElse(ScoreFeature, None)).reverse.take(MaxCommunityCandidates) val updatedCandidates = otherCandidates ++ filteredCandidates SelectorResult(remainingCandidates = updatedCandidates, result = result) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "finagle/finagle-mysql/src/main/scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/simclusters_features", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "kafka/finagle-kafka/finatra-kafka/src/main/scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweet_mixer", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/communities", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature/location", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", "servo/repo/src/main/scala", "src/scala/com/twitter/timelines/prediction/common/adapters:base", "src/scala/com/twitter/timelines/prediction/features/common", "src/thrift/com/twitter/timelines/served_candidates_logging:served_candidates_logging-scala", "src/thrift/com/twitter/timelines/suggests/common:data_record_metadata-scala", "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", "src/thrift/com/twitter/timelines/timeline_logging:thrift-scala", "strato/config/columns/home-mixer:home-mixer-strato-client", "strato/config/columns/videoRecommendations/twitterClip:twitterClip-strato-client", "strato/config/src/thrift/com/twitter/strato/columns/content_understanding:content_understanding-scala", "timelines/data_processing/jobs/light_ranking/light_ranking_features_prep:recap_partial_features_for_two_tower_models", "timelines/ml:kafka", "timelines/ml:pldr-client", "timelines/ml:pldr-conversion", "timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding", "user_history_transformer/service/src/main/java/com/x/user_action_sequence", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CacheCandidateFeaturesSideEffect.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.side_effect import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.side_effect.BaseCacheCandidateFeaturesSideEffect import com.twitter.home_mixer.param.HomeMixerFlagName.DataRecordMetadataStoreConfigsYmlFlag import com.twitter.home_mixer.param.HomeMixerInjectionNames._ import com.twitter.inject.annotations.Flag import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding import com.twitter.storehaus.ReadableStore import com.twitter.storehaus.Store import com.twitter.strato.generated.client.videoRecommendations.twitterClip.TwitterClipEmbeddingMhClientColumn import com.twitter.timelines.served_candidates_logging.{thriftscala => sc} import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pdr} import com.twitter.twistly.thriftscala.VideoViewEngagementType import com.twitter.twistly.thriftscala.WatchTimeMetadata import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class CacheCandidateFeaturesSideEffect @Inject() ( @Flag(DataRecordMetadataStoreConfigsYmlFlag) dataRecordMetadataStoreConfigsYml: String, @Named(MemcacheCandidateFeaturesStore) store: Store[ sc.CandidateFeatureKey, pdr.PolyDataRecord ], @Named(TweetWatchTimeMetadataStore) tweetWatchTimeMetadataStore: ReadableStore[ (Long, VideoViewEngagementType), WatchTimeMetadata ], twitterClipEmbeddingMhClientColumn: TwitterClipEmbeddingMhClientColumn, @Named(TwhinVideoEmbeddingsStore) twhinVideoStore: ReadableStore[Long, TwhinTweetEmbedding], statsReceiver: StatsReceiver) extends BaseCacheCandidateFeaturesSideEffect( dataRecordMetadataStoreConfigsYml, store, tweetWatchTimeMetadataStore, twitterClipEmbeddingMhClientColumn, twhinVideoStore, statsReceiver) { override val identifier: SideEffectIdentifier = SideEffectIdentifier("CacheCandidateFeatures") override val statScope: String = getClass.getSimpleName } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CacheRequestInfoSideEffect.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.side_effect import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature import com.twitter.home_mixer.model.PredictedFavoriteScoreFeature import com.twitter.home_mixer.model.PredictedReplyScoreFeature import com.twitter.home_mixer.model.PredictedRetweetScoreFeature import com.twitter.home_mixer.model.PredictedShareScoreFeature import com.twitter.home_mixer.model.PredictedVideoQualityViewScoreFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithLongTimeout import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableCacheRequestInfoParam import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.HasMarshalling import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.client.Client import com.twitter.strato.generated.client.home_mixer.StoreRequestInfoClientColumn import com.twitter.strato.generated.client.home_mixer.StoreRequestInfoClientColumn.FavAndRetweetAndReplyAndShareAndVqv import com.twitter.strato.generated.client.home_mixer.StoreRequestInfoClientColumn.PostIdAndHeavyRankerScores import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class CacheRequestInfoSideEffect @Inject() ( @Named(BatchedStratoClientWithLongTimeout) stratoClient: Client, serviceIdentifier: ServiceIdentifier) extends PipelineResultSideEffect[PipelineQuery, HasMarshalling] with Conditionally[PipelineQuery, HasMarshalling] { override val identifier: SideEffectIdentifier = SideEffectIdentifier("CacheRequestInfo") private val isProdEnv = serviceIdentifier.environment == "prod" private val retrievalSignalExecutor = new StoreRequestInfoClientColumn(stratoClient).executer override def onlyIf( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: HasMarshalling ): Boolean = query.params(EnableCacheRequestInfoParam) && isProdEnv override def apply( inputs: PipelineResultSideEffect.Inputs[PipelineQuery, HasMarshalling] ): Stitch[Unit] = { val posts = inputs.selectedCandidates.collect { case candidate if !candidate.features.getOrElse(IsReadFromCacheFeature, false) => PostIdAndHeavyRankerScores( postId = candidate.candidateIdLong, heavyRankerScores = Some( FavAndRetweetAndReplyAndShareAndVqv( fav = candidate.features.getOrElse(PredictedFavoriteScoreFeature, None), retweet = candidate.features.getOrElse(PredictedRetweetScoreFeature, None), reply = candidate.features.getOrElse(PredictedReplyScoreFeature, None), share = candidate.features.getOrElse(PredictedShareScoreFeature, None), vqv = candidate.features.getOrElse(PredictedVideoQualityViewScoreFeature, None) ) ) ) } val arg = StoreRequestInfoClientColumn.Arg( userId = inputs.query.getRequiredUserId, posts = posts, requestTimestampMs = inputs.query.queryTime.inMilliseconds ) retrievalSignalExecutor.execute(arg) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CacheRetrievalSignalSideEffect.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.side_effect import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SourceSignalFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.BatchedStratoClientWithLongTimeout import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableCacheRetrievalSignalParam import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.HasMarshalling import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.strato.client.Client import com.twitter.strato.client.Putter import com.twitter.strato.generated.client.home_mixer.RetrievalSignalv2ClientColumn import com.twitter.usersignalservice.{thriftscala => se} import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class CacheRetrievalSignalSideEffect @Inject() ( @Named(BatchedStratoClientWithLongTimeout) stratoClient: Client, serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver) extends PipelineResultSideEffect[PipelineQuery, HasMarshalling] with Conditionally[PipelineQuery, HasMarshalling] { override val identifier: SideEffectIdentifier = SideEffectIdentifier("CacheRetrievalSignal") private val scopedStats = statsReceiver.scope(getClass.getSimpleName) private val isProdEnv = serviceIdentifier.environment == "prod" private val retrievalSignalPutter: Putter[(Long, Long), hmt.RetrievalSignal] = new RetrievalSignalv2ClientColumn(stratoClient).putter override def onlyIf( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: HasMarshalling ): Boolean = query.params(EnableCacheRetrievalSignalParam) && isProdEnv override def apply( inputs: PipelineResultSideEffect.Inputs[PipelineQuery, HasMarshalling] ): Stitch[Unit] = { scopedStats.counter("side_effect_applied").incr(1) val sourceSignalWrites = inputs.selectedCandidates.map { candidate => scopedStats.counter("total_candidates_processed").incr(1) val userId = inputs.query.getRequiredUserId val signalFeature = candidate.features.getOrElse(SourceSignalFeature, None) val servedType = candidate.features.get(ServedTypeFeature) val retrievalSignalOpt = (signalFeature, servedType) match { case (Some(signal), _) if signal.id > 0L && signal.signalEntity.isDefined => scopedStats.counter("valid_existing_signal_feature").incr(1) signal.signalEntity.flatMap { entity => if (entity != se.SignalEntity.User || signal.id != userId) { Some( hmt.RetrievalSignal( signalId = signal.id, signalEntity = entity, authorId = signal.authorId.filter(_ > 0L) )) } else None } case (None, hmt.ServedType.ForYouInNetwork) => scopedStats.counter("valid_in_network_signal_feature").incr(1) candidate.features.get(AuthorIdFeature).map { authorId => hmt.RetrievalSignal( signalId = authorId, signalEntity = se.SignalEntity.User, authorId = None ) } case (None, hmt.ServedType.ForYouUteg) => scopedStats.counter("valid_uteg_liked_by_signal_feature").incr(1) val validLikedByUserIds = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) validLikedByUserIds.headOption.map { lastLikedByUserId => hmt.RetrievalSignal( signalId = lastLikedByUserId, signalEntity = se.SignalEntity.User, authorId = None ) } case _ => scopedStats.counter("missing_or_invalid_signal_feature").incr(1) None } retrievalSignalOpt .map { retrievalSignal => val sourceTweetId: Long = candidate.features .getOrElse(SourceTweetIdFeature, None).getOrElse(candidate.candidateIdLong) retrievalSignalPutter.put((userId, sourceTweetId), retrievalSignal) }.getOrElse(Stitch.Unit) }.toSeq Stitch.collect(sourceSignalWrites).map(_ => ()) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CachedScoredTweetsSideEffect.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.side_effect import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.transport.Transport import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.model.PredictedFavoriteScoreFeature import com.twitter.home_mixer.model.PredictedReplyScoreFeature import com.twitter.home_mixer.model.PredictedRetweetScoreFeature import com.twitter.home_mixer.model.PredictedShareScoreFeature import com.twitter.home_mixer.model.PredictedVideoQualityViewScoreFeature import com.twitter.home_mixer.model.PredictedDwellScoreFeature import com.twitter.home_mixer.model.PredictedNegativeFeedbackV2ScoreFeature import com.twitter.home_mixer.model.PredictedGoodClickConvoDescUamGt2ScoreFeature import com.twitter.home_mixer.model.PredictedGoodProfileClickScoreFeature import com.twitter.home_mixer.model.PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature import com.twitter.home_mixer.model.PredictedReplyEngagedByAuthorScoreFeature import com.twitter.home_mixer.param.HomeMixerInjectionNames.ScoredTweetsCache import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityIdFeature import com.twitter.product_mixer.component_library.feature.communities.CommunitiesSharedFeatures.CommunityNameFeature import com.twitter.product_mixer.component_library.feature.location.LocationSharedFeatures.LocationIdFeature import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.TopicContextFunctionalityTypeMarshaller import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.cache.TtlCache import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton @Singleton class CachedScoredTweetsSideEffect @Inject() ( @Named(ScoredTweetsCache) scoredTweetsCache: TtlCache[Long, hmt.ScoredTweetsResponse]) extends PipelineResultSideEffect[PipelineQuery, ScoredTweetsResponse] with Conditionally[PipelineQuery, ScoredTweetsResponse] { override val identifier: SideEffectIdentifier = SideEffectIdentifier("CachedScoredTweets") private val MaxTweetsToCache = 1000 override def onlyIf( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: ScoredTweetsResponse ): Boolean = { val serviceIdentifier = ServiceIdentifier.fromCertificate(Transport.peerCertificate) serviceIdentifier.role != "explore-mixer" && serviceIdentifier.role != "video-mixer" } def buildCachedScoredTweets( query: PipelineQuery, candidates: Seq[CandidateWithDetails] ): hmt.ScoredTweetsResponse = { val tweets = candidates.map { candidate => val sgsValidLikedByUserIds = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) val validLikedByUserIds = candidate.features.getOrElse(ValidLikedByUserIdsFeature, Seq.empty) val sgsValidFollowedByUserIds = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty) val ancestors = candidate.features.getOrElse(AncestorsFeature, Seq.empty) val mediaIds = candidate.features.getOrElse(TweetMediaIdsFeature, Seq.empty) val sourceSignalOpt = candidate.features.getOrElse(SourceSignalFeature, None) val sourceSignal = sourceSignalOpt.map { signal => hmt.SourceSignal( id = signal.id, signalType = signal.signalType, signalEntity = signal.signalEntity, authorId = signal.authorId, ) } val predictedScores = hmt.PredictedScores( favoriteScore = candidate.features.getOrElse(PredictedFavoriteScoreFeature, None), replyScore = candidate.features.getOrElse(PredictedReplyScoreFeature, None), retweetScore = candidate.features.getOrElse(PredictedRetweetScoreFeature, None), replyEngagedByAuthorScore = candidate.features.getOrElse(PredictedReplyEngagedByAuthorScoreFeature, None), goodClickConvoDescFavoritedOrRepliedScore = candidate.features .getOrElse(PredictedGoodClickConvoDescFavoritedOrRepliedScoreFeature, None), goodClickConvoDescUamGt2Score = candidate.features.getOrElse(PredictedGoodClickConvoDescUamGt2ScoreFeature, None), goodProfileClickScore = candidate.features.getOrElse(PredictedGoodProfileClickScoreFeature, None), videoQualityViewScore = candidate.features.getOrElse(PredictedVideoQualityViewScoreFeature, None), shareScore = candidate.features.getOrElse(PredictedShareScoreFeature, None), dwellScore = candidate.features.getOrElse(PredictedDwellScoreFeature, None), negativeFeedbackV2Score = candidate.features.getOrElse(PredictedNegativeFeedbackV2ScoreFeature, None) ) hmt.ScoredTweet( tweetId = candidate.candidateIdLong, authorId = candidate.features.get(AuthorIdFeature).get, // Cache the model score instead of the final score because rescoring is per-request score = candidate.features.getOrElse(WeightedModelScoreFeature, None), servedType = candidate.features.get(ServedTypeFeature), sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None), quotedTweetId = candidate.features.getOrElse(QuotedTweetIdFeature, None), quotedUserId = candidate.features.getOrElse(QuotedUserIdFeature, None), inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None), inReplyToUserId = candidate.features.getOrElse(InReplyToUserIdFeature, None), directedAtUserId = candidate.features.getOrElse(DirectedAtUserIdFeature, None), inNetwork = Some(candidate.features.getOrElse(InNetworkFeature, true)), sgsValidLikedByUserIds = Some(sgsValidLikedByUserIds), validLikedByUserIds = Some(validLikedByUserIds), sgsValidFollowedByUserIds = Some(sgsValidFollowedByUserIds), topicId = candidate.features.getOrElse(TopicIdSocialContextFeature, None), topicFunctionalityType = candidate.features .getOrElse(TopicContextFunctionalityTypeFeature, None).map( TopicContextFunctionalityTypeMarshaller(_)), ancestors = if (ancestors.nonEmpty) Some(ancestors) else None, isReadFromCache = Some(true), exclusiveConversationAuthorId = candidate.features .getOrElse(ExclusiveConversationAuthorIdFeature, None), authorMetadata = Some( hmt.AuthorMetadata( blueVerified = candidate.features.getOrElse(AuthorIsBlueVerifiedFeature, false), goldVerified = candidate.features.getOrElse(AuthorIsGoldVerifiedFeature, false), grayVerified = candidate.features.getOrElse(AuthorIsGrayVerifiedFeature, false), legacyVerified = candidate.features.getOrElse(AuthorIsLegacyVerifiedFeature, false), creator = candidate.features.getOrElse(AuthorIsCreatorFeature, false), followers = candidate.features.getOrElse(AuthorFollowersFeature, None) )), lastScoredTimestampMs = candidate.features .getOrElse(LastScoredTimestampMsFeature, Some(query.queryTime.inMilliseconds)), candidatePipelineIdentifier = candidate.features .getOrElse(CachedCandidatePipelineIdentifierFeature, Some(candidate.source.name)), tweetUrls = Some(candidate.features.getOrElse(TweetUrlsFeature, Seq.empty)), perspectiveFilteredLikedByUserIds = None, predictionRequestId = candidate.features.getOrElse(PredictionRequestIdFeature, None), communityId = candidate.features.getOrElse(CommunityIdFeature, None), communityName = candidate.features.getOrElse(CommunityNameFeature, None), listId = candidate.features.getOrElse(ListIdFeature, None), listName = candidate.features.getOrElse(ListNameFeature, None), tweetTypeMetrics = candidate.features.getOrElse(TweetTypeMetricsFeature, None), debugString = candidate.features.getOrElse(DebugStringFeature, None), viralContentCreator = Some(candidate.features.getOrElse(ViralContentCreatorFeature, false)), locationId = candidate.features.getOrElse(LocationIdFeature, None), isArticle = Some(candidate.features.getOrElse(IsArticleFeature, false)), hasVideo = Some(candidate.features.getOrElse(HasVideoFeature, false)), videoDurationMs = candidate.features.getOrElse(VideoDurationMsFeature, None), mediaIds = if (mediaIds.nonEmpty) Some(mediaIds) else None, grokAnnotations = candidate.features.getOrElse(GrokAnnotationsFeature, None), predictedScores = Some(predictedScores), tweetMixerScore = candidate.features.getOrElse(TweetMixerScoreFeature, None), clipClusterIdsFeature = Some( hmt.ClipClusterIdsFeature( tweetMediaClusterIdsFeature = Some( candidate.features.getOrElse(TweetMediaClusterIdsFeature, Map.empty[Long, Long])), clipImageClusterIdsFeature = Some(candidate.features.getOrElse(ClipImageClusterIdsFeature, Map.empty[Long, Long])) )), grokSlopScoreFeature = candidate.features.getOrElse(GrokSlopScoreFeature, None), mediaCompletionRate = candidate.features.getOrElse(TweetMediaCompletionRateFeature, None), tweetText = candidate.features.getOrElse(TweetTextFeature, None), sourceSignal = sourceSignal, grokContentCreator = Some(candidate.features.getOrElse(GrokContentCreatorFeature, false)), gorkContentCreator = Some(candidate.features.getOrElse(GorkContentCreatorFeature, false)), // MultiModalEmbeddingsFeature are not cached because the value size is too large for memcache multiModalEmbedding = None ) } hmt.ScoredTweetsResponse(tweets) } final override def apply( inputs: PipelineResultSideEffect.Inputs[PipelineQuery, ScoredTweetsResponse] ): Stitch[Unit] = { val candidates = (inputs.selectedCandidates ++ inputs.remainingCandidates ++ inputs.droppedCandidates) .filter(_.features.getOrElse(ScoreFeature, None).exists(_ > 0.0)) val truncatedCandidates = if (candidates.size > MaxTweetsToCache) candidates .sortBy(-_.features.getOrElse(ScoreFeature, None).getOrElse(0.0)).take(MaxTweetsToCache) else candidates if (truncatedCandidates.nonEmpty) { val ttl = inputs.query.params(CachedScoredTweets.TTLParam) val scoredTweets = buildCachedScoredTweets(inputs.query, truncatedCandidates) Stitch.callFuture(scoredTweetsCache.set(inputs.query.getRequiredUserId, scoredTweets, ttl)) } else Stitch.Unit } override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.4) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/CommonFeaturesSideEffect.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.side_effect import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finatra.kafka.serde.ScalaSerdes import com.twitter.home_mixer.functional_component.side_effect.CommonFeaturesPldrConverter import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeFeaturesFlag import com.twitter.home_mixer.param.HomeMixerInjectionNames.CommonFeaturesScribeEventPublisher import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.side_effect.KafkaAndScribePublishingSideEffect import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.HasMarshalling import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.ml.kafka.serde.TBaseSerde import com.twitter.timelines.suggests.common.poly_data_record.thriftjava.PolyDataRecord import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.common.serialization.Serializer /** * Publish common features sent to prediction service + some other features as PLDR format * into both scribe logs and kafka */ @Singleton class CommonFeaturesSideEffect @Inject() ( serviceIdentifier: ServiceIdentifier, commonFeaturesPldrConverter: CommonFeaturesPldrConverter, @Flag(ScribeFeaturesFlag) enableScribeFeatures: Boolean, @Named(CommonFeaturesScribeEventPublisher) override val logPipelinePublisher: EventPublisher[ pldr.PolyDataRecord ]) extends KafkaAndScribePublishingSideEffect[ Long, pldr.PolyDataRecord, PipelineQuery, HasMarshalling ] with Conditionally[PipelineQuery, HasMarshalling] { override val identifier: SideEffectIdentifier = SideEffectIdentifier("CommonFeaturesSideEffect") private val kafkaTopic: String = serviceIdentifier.environment.toLowerCase match { case "prod" => "tq_ct_common_features" case _ => "tq_ct_common_features_staging" } override val bootstrapServer: String = "/s/kafka/timeline:kafka-tls" override val keySerde: Serializer[Long] = ScalaSerdes.Long.serializer() override val valueSerde: Serializer[PolyDataRecord] = TBaseSerde.Thrift[pldr.PolyDataRecord]().serializer override val clientId: String = "home_mixer_common_features_producer" override def enableScribePublishing(query: PipelineQuery): Boolean = enableScribeFeatures /** @see [[common.Conditionally.onlyIf]] */ override def onlyIf( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: HasMarshalling ): Boolean = selectedCandidates.nonEmpty /** * Build the record to be published to Kafka from query, selections and response * * @param query PipelineQuery * @param selectedCandidates Result after Selectors are executed * @param remainingCandidates Candidates which were not selected * @param droppedCandidates Candidates dropped during selection * @param response Result after Unmarshalling * * @return A sequence of to-be-published ProducerRecords */ override def buildRecords( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: HasMarshalling ): Seq[ProducerRecord[Long, PolyDataRecord]] = { commonFeaturesPldrConverter .getCommonFeaturesPldr(query, selectedCandidates).map { case (predictionRequestId, pldr) => new ProducerRecord(kafkaTopic, predictionRequestId, pldr) }.toSeq } override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(98.5)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredCandidateFeatureKeysKafkaSideEffect.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.side_effect import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.mysql.Client import com.twitter.finagle.mysql.Transactions import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.util.DefaultTimer import com.twitter.finatra.kafka.serde.ScalaSerdes import com.twitter.home_mixer.functional_component.scorer.CandidateFeaturesDataRecordFeature import com.twitter.home_mixer.model.HomeFeatures.GrokAnnotationsFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature import com.twitter.home_mixer.model.PredictedScoreFeature.PredictedScoreFeatureSet import com.twitter.home_mixer.param.HomeGlobalParams.IsSelectedByHeavyRankerCountParam import com.twitter.home_mixer.param.HomeMixerFlagName.DataRecordMetadataStoreConfigsYmlFlag import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableScoredCandidateFeatureKeysKafkaPublishingParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ScribedScoredCandidateNumParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.inject.annotations.Flag import com.twitter.ml.api.DataRecordMerger import com.twitter.product_mixer.component_library.side_effect.KafkaPublishingSideEffect import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter import com.twitter.product_mixer.core.feature.featuremap.datarecord.SpecificFeatures import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.HasMarshalling import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.CandidateAndCommonFeaturesStreamingUtils import com.twitter.timelines.data_processing.jobs.light_ranking.light_ranking_features_prep.RecapPartialFeaturesForTwoTowerModels import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.ScoredCandidateFeatureKeysAdapter import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.ScoredCandidateFeatureKeysFields import com.twitter.timelines.ml.kafka.serde.TBaseSerde import com.twitter.timelines.ml.pldr.client.MysqlClientUtils import com.twitter.timelines.ml.pldr.client.VersionedMetadataCacheClient import com.twitter.timelines.ml.pldr.conversion.VersionIdAndFeatures import com.twitter.timelines.suggests.common.data_record_metadata.{thriftscala => drmd} import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} import com.twitter.util.Try import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.common.serialization.Serializer import scala.collection.JavaConverters._ /** * Pipeline side-effect that publishes scored candidate feature keys to a Kafka topic. */ @Singleton class ScoredCandidateFeatureKeysKafkaSideEffect @Inject() ( serviceIdentifier: ServiceIdentifier, @Flag(DataRecordMetadataStoreConfigsYmlFlag) dataRecordMetadataStoreConfigsYml: String, statsReceiver: StatsReceiver) extends KafkaPublishingSideEffect[ Long, pldr.PolyDataRecord, PipelineQuery, HasMarshalling ] with Conditionally[PipelineQuery, HasMarshalling] with Logging { override val identifier: SideEffectIdentifier = SideEffectIdentifier( "ScoredCandidateFeatureKeysKafka") private val statScope: String = this.getClass.getSimpleName private val scopedStatsReceiver = statsReceiver.scope(statScope) private val metadataFetchFailedCounter = scopedStatsReceiver.counter("metadataFetchFailed") private val randomSelectedCandidatesStat = scopedStatsReceiver.stat("randomSelectedCandidates") private val randomDroppedCandidatesStat = scopedStatsReceiver.stat("randomDroppedCandidates") override def onlyIf( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: HasMarshalling ): Boolean = { if (query.params.getBoolean(EnableScoredCandidateFeatureKeysKafkaPublishingParam)) { val scribedCandidateNumber = query.params(ScribedScoredCandidateNumParam) filterRankedCandidates(selectedCandidates).size >= scribedCandidateNumber && filterRankedCandidates(droppedCandidates).size >= scribedCandidateNumber && selectedCandidates .forall(!_.features.getOrElse(IsReadFromCacheFeature, false)) && remainingCandidates .forall(!_.features.getOrElse(IsReadFromCacheFeature, false)) && droppedCandidates .forall(!_.features.getOrElse(IsReadFromCacheFeature, false)) } else false } private val kafkaTopic: String = serviceIdentifier.environment.toLowerCase match { case "prod" => "home_mixer_dropped_candidates_features" case _ => "deep_retrieval_candidates_data_staging" } override val bootstrapServer: String = "/s/kafka/timeline:kafka-tls" override val keySerde: Serializer[Long] = ScalaSerdes.Long.serializer() override val valueSerde: Serializer[pldr.PolyDataRecord] = TBaseSerde.Thrift[pldr.PolyDataRecord]().serializer override val clientId: String = "home_mixer_dropped_candidate_feature_keys_producer" lazy private val dataRecordMetadataStoreClient: Option[Client with Transactions] = Try { try { val c = MysqlClientUtils.parseConfigFromYaml(dataRecordMetadataStoreConfigsYml) logger.info(s"pldr mysql config: ${c.host} ${c.port} ${c.user} ${c.database}") } catch { case e: Throwable => logger.error("pldr mysql error: " + e.toString) } MysqlClientUtils.mysqlClientProvider( MysqlClientUtils.parseConfigFromYaml(dataRecordMetadataStoreConfigsYml) ) }.toOption lazy private val versionedMetadataCacheClientOpt: Option[ VersionedMetadataCacheClient[Map[drmd.FeaturesCategory, Option[VersionIdAndFeatures]]] ] = dataRecordMetadataStoreClient.map { mysqlClient => new VersionedMetadataCacheClient[Map[drmd.FeaturesCategory, Option[VersionIdAndFeatures]]]( maximumSize = 1, expireDurationOpt = None, mysqlClient = mysqlClient, transform = CandidateAndCommonFeaturesStreamingUtils.metadataTransformer, statsReceiver = statsReceiver ) } versionedMetadataCacheClientOpt.foreach { _.metadataFetchTimerTask( CandidateAndCommonFeaturesStreamingUtils.metadataFetchKey, metadataFetchTimer = DefaultTimer, metadataFetchInterval = 90.seconds, metadataFetchFailedCounter = metadataFetchFailedCounter ) } private val drMerger = new DataRecordMerger private val predictedScoreFeaturesDataRecordAdapter = new DataRecordConverter(SpecificFeatures(PredictedScoreFeatureSet)) override def buildRecords( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: HasMarshalling ): Seq[ProducerRecord[Long, pldr.PolyDataRecord]] = { val rankedSelected = filterRankedCandidates(selectedCandidates) .filterNot { candidate => val annotations = candidate.features.getOrElse(GrokAnnotationsFeature, None) val isNsfw = annotations.flatMap(_.metadata.map(_.isNsfw)).getOrElse(false) val isSoftNsfw = annotations.flatMap(_.metadata.map(_.isSoftNsfw)).getOrElse(false) val isGore = annotations.flatMap(_.metadata.map(_.isGore)).getOrElse(false) val isViolent = annotations.flatMap(_.metadata.map(_.isViolent)).getOrElse(false) val isSpam = annotations.flatMap(_.metadata.map(_.isSpam)).getOrElse(false) isNsfw || isSoftNsfw || isGore || isViolent || isSpam } val rankedDropped = filterRankedCandidates(droppedCandidates) val candidates = rankedSelected ++ rankedDropped val isSelectedCandidateIds: Set[Long] = selectedCandidates.map(_.candidateIdLong).toSet val candidatesHeavyRankerScoreBasedRank: Map[Long, Int] = candidates .sortBy( -_.features .getOrElse(WeightedModelScoreFeature, None).getOrElse(Double.NegativeInfinity)).map( _.candidateIdLong).zipWithIndex.toMap val isSelectedByHeavyRankerCount = query.params(IsSelectedByHeavyRankerCountParam) val predictionRequestId = candidates.headOption.flatMap { candidate => candidate.features.getOrElse(PredictionRequestIdFeature, None) } val scribedCandidateNumber = query.params(ScribedScoredCandidateNumParam) val randomRankedSelected = selectRandomCandidates(rankedSelected, scribedCandidateNumber) val randomRankedDropped = selectRandomCandidates(rankedDropped, scribedCandidateNumber) randomSelectedCandidatesStat.add(randomRankedSelected.size) randomDroppedCandidatesStat.add(randomRankedDropped.size) val randomCandidates = randomRankedSelected ++ randomRankedDropped randomCandidates.flatMap { candidate => val key = candidate.candidateIdLong try { val scoredCandidateFeatureKeysDataRecord = ScoredCandidateFeatureKeysAdapter .adaptToDataRecords( ScoredCandidateFeatureKeysFields( viewerId = query.getRequiredUserId, tweetId = key, isSelected = isSelectedCandidateIds.contains(candidate.candidateIdLong), isSelectedByHeavyRanker = candidatesHeavyRankerScoreBasedRank .getOrElse( candidate.candidateIdLong, candidates.size) < isSelectedByHeavyRankerCount, predictionRequestId = predictionRequestId, requestTimeMs = Some(query.queryTime.inMilliseconds), scribedCandidateNumber = scribedCandidateNumber, viewerFollowsOriginalAuthor = Some(candidate.features.getOrElse(InNetworkFeature, false)), rankByHeavyRanker = Some(candidatesHeavyRankerScoreBasedRank.getOrElse(key, candidates.size)) ) ).asScala.head val candidateFeaturesDataRecord = CandidateAndCommonFeaturesStreamingUtils .extractFeaturesFromDrWithContext( candidate.features.get(CandidateFeaturesDataRecordFeature), RecapPartialFeaturesForTwoTowerModels.scoredCandidateFeatureContext ) drMerger.merge(scoredCandidateFeatureKeysDataRecord, candidateFeaturesDataRecord) val predictedScoreFeaturesDataRecord = predictedScoreFeaturesDataRecordAdapter.toDataRecord(candidate.features) drMerger.merge(scoredCandidateFeatureKeysDataRecord, predictedScoreFeaturesDataRecord) val convertedPlDrOpt = CandidateAndCommonFeaturesStreamingUtils.candidateFeaturesToPolyDataRecord( versionedMetadataCacheClientOpt = versionedMetadataCacheClientOpt, candidateFeatures = scoredCandidateFeatureKeysDataRecord, valueFormat = pldr.PolyDataRecord._Fields.LITE_COMPACT_DATA_RECORD ) convertedPlDrOpt.map { convertedPlDr => new ProducerRecord(kafkaTopic, key, convertedPlDr) } } catch { case e: Exception => logger.error( s"Error while converting features to PolyDataRecord for candidateId: $key", e ) None } } } private def filterRankedCandidates( candidates: Seq[CandidateWithDetails] ): Seq[CandidateWithDetails] = candidates.filter(_.features.contains(CandidateFeaturesDataRecordFeature)) private def selectRandomCandidates( candidates: Seq[CandidateWithDetails], count: Int ): Seq[CandidateWithDetails] = scala.util.Random.shuffle(CandidatesUtil.getItemCandidates(candidates)).take(count) override val alerts = Seq( HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(98.5) ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredContentExplorationCandidateScoreFeatureKafkaSideEffect.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.side_effect import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.finatra.kafka.serde.ScalaSerdes import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableContentExplorationScoreScribingParam import com.twitter.product_mixer.component_library.side_effect.KafkaPublishingSideEffect import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.HasMarshalling import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.strato.columns.content_understanding.content_understanding.thriftscala.ContentExplorationServingResult import com.twitter.strato.columns.content_understanding.content_understanding.thriftscala.ContentExplorationServingResultsAnalysis import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.common.serialization.Serializer /** * Pipeline side-effect that publishes scores feature keys to a Kafka topic. */ @Singleton class ScoredContentExplorationCandidateScoreFeatureKafkaSideEffect @Inject() ( serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver) extends KafkaPublishingSideEffect[ Long, ContentExplorationServingResultsAnalysis, PipelineQuery, HasMarshalling ] with Conditionally[PipelineQuery, HasMarshalling] with Logging { override val identifier: SideEffectIdentifier = SideEffectIdentifier( "ScoredContentExplorationCandidateScoreFeatureKafka") private val statScope: String = this.getClass.getSimpleName private val scopedStatsReceiver = statsReceiver.scope(statScope) private val failedScribingCount = scopedStatsReceiver.scope("failed").counter() override def onlyIf( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: HasMarshalling ): Boolean = query.params(EnableContentExplorationScoreScribingParam) private val eligibleServedType: Set[hmt.ServedType] = Set( hmt.ServedType.ForYouContentExploration, hmt.ServedType.ForYouContentExplorationTier2, hmt.ServedType.ForYouContentExplorationDeepRetrievalI2i, hmt.ServedType.ForYouContentExplorationTier2DeepRetrievalI2i, hmt.ServedType.ForYouUserInterestSummary ) private val kafkaTopic: String = serviceIdentifier.environment.toLowerCase match { case "prod" => "content_exploration_ranking_analysis" case _ => "content_exploration_ranking_analysis_devel" } override val bootstrapServer: String = "" override val keySerde: Serializer[Long] = ScalaSerdes.Long.serializer() override val valueSerde: Serializer[ContentExplorationServingResultsAnalysis] = ScalaSerdes.Thrift[ContentExplorationServingResultsAnalysis].serializer override val clientId: String = "home_mixer_dropped_candidate_feature_keys_producer" override def buildRecords( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: HasMarshalling ): Seq[ProducerRecord[Long, ContentExplorationServingResultsAnalysis]] = { try { val userId: Long = query.getUserOrGuestId.getOrElse(-1L) val allCandidates = selectedCandidates ++ remainingCandidates ++ droppedCandidates val eligible = allCandidates.filter(selectEligibleCandidates) val scoredResults: Seq[ContentExplorationServingResult] = eligible.map { cdd => ContentExplorationServingResult( cdd.candidateIdLong, cdd.features.get(ScoreFeature), cdd.features.get(WeightedModelScoreFeature), Some(cdd.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined).toString) ) } val analysisResult = ContentExplorationServingResultsAnalysis( userId, scoredResults ) Seq( new ProducerRecord[Long, ContentExplorationServingResultsAnalysis]( kafkaTopic, userId, analysisResult ) ) } catch { case _: Throwable => failedScribingCount.incr() Seq.empty } } def selectEligibleCandidates( candidate: CandidateWithDetails ): Boolean = { val servedType = candidate.features .getOrElse(ServedTypeFeature, hmt.ServedType.Undefined) eligibleServedType.contains(servedType) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredPhoenixCandidatesKafkaSideEffect.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.side_effect import com.google.inject.name.Named import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.finatra.kafka.serde.ScalaSerdes import com.twitter.finatra.kafka.serde.UnKeyed import com.twitter.finatra.kafka.serde.UnKeyedSerde import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.model.PhoenixPredictedScoreFeature.PhoenixPredictedScoreFeatures import com.twitter.home_mixer.model.PredictedScoreFeature.PredictedScoreFeatures import com.twitter.home_mixer.param.HomeGlobalParams.PhoenixCluster import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableScoredPhoenixCandidatesKafkaSideEffectParam import com.twitter.home_mixer.util.PhoenixUtils.createCandidateSets import com.twitter.home_mixer.util.PhoenixUtils.getPredictionResponseMap import com.twitter.home_mixer.util.PhoenixUtils.getTweetInfoFromCandidates import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.side_effect.KafkaPublishingSideEffect import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect.Conditionally import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.marshalling.HasMarshalling import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.util.MemoizingStatsReceiver import com.twitter.stitch.Stitch import com.twitter.timelines.timeline_logging.thriftscala.PredictionScore import com.twitter.timelines.timeline_logging.thriftscala.ScoredCandidate import com.twitter.util.logging.Logging import com.x.user_action_sequence.ActionName import com.x.user_action_sequence.PredictNextActionsRequest import io.grpc.ManagedChannel import javax.inject.Inject import javax.inject.Singleton import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.common.serialization.Serializer /** * Pipeline side-effect that publishes scored phoenix candidates to a Kafka topic. */ @Singleton class ScoredPhoenixCandidatesKafkaSideEffect @Inject() ( @Named("PhoenixClient") channelsMap: Map[PhoenixCluster.Value, Seq[ManagedChannel]], serviceIdentifier: ServiceIdentifier, statsReceiver: StatsReceiver) extends KafkaPublishingSideEffect[ UnKeyed, ScoredCandidate, PipelineQuery, HasMarshalling ] with Conditionally[PipelineQuery, HasMarshalling] with Logging { override val identifier: SideEffectIdentifier = SideEffectIdentifier( "ScoredPhoenixCandidatesKafka") private val statScope: String = this.getClass.getSimpleName val memoizingStatsReceiver: MemoizingStatsReceiver = new MemoizingStatsReceiver( statsReceiver.scope(statScope)) override def onlyIf( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: HasMarshalling ): Boolean = { query.params(EnableScoredPhoenixCandidatesKafkaSideEffectParam) && query.features.flatMap(_.getOrElse(UserActionsFeature, None)).isDefined } private val kafkaTopic: String = serviceIdentifier.environment.toLowerCase match { case "prod" => "home_mixer_phoenix_scored_candidates" case _ => "home_mixer_phoenix_scored_candidates_staging" } override val bootstrapServer: String = "" override val keySerde: Serializer[UnKeyed] = UnKeyedSerde.serializer() override val valueSerde: Serializer[ScoredCandidate] = ScalaSerdes.Thrift[ScoredCandidate].serializer override val clientId: String = "home_mixer_phoenix_scored_candidate_producer" // Returns Map from phoenixCluster -> Map [Tweets -> Map [Actions -> Score]] private def getPredictionResponsesAllClusters( request: PredictNextActionsRequest ): Stitch[Map[String, Map[Long, Map[ActionName, Double]]]] = { val phoenixClusters = channelsMap.keySet.toSeq val timeoutMs = 1000 Stitch .traverse(phoenixClusters) { phoenixCluster => val channels = channelsMap(phoenixCluster) getPredictionResponseMap( request, channels, phoenixCluster.toString, timeoutMs, memoizingStatsReceiver) .handle { case _ => Map.empty[Long, Map[ActionName, Double]] } .map(phoenixCluster.toString -> _) }.map(_.toMap) } // Not defined buildRecords as we override apply to use buildRecordsStitch override def buildRecords( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: HasMarshalling ): Seq[ProducerRecord[UnKeyed, ScoredCandidate]] = { ??? } def buildRecordsStitch( query: PipelineQuery, selectedCandidates: Seq[CandidateWithDetails], ): Stitch[Seq[ProducerRecord[UnKeyed, ScoredCandidate]]] = { val nonCachedSelectedCandidates = selectedCandidates.filterNot(_.features.getOrElse(IsReadFromCacheFeature, false)) val candidates = nonCachedSelectedCandidates.map(_.getCandidate[TweetCandidate]()) val featureMaps = nonCachedSelectedCandidates.map(_.features) val tweetInfos = getTweetInfoFromCandidates(candidates, featureMaps) val request = createCandidateSets(query, tweetInfos) val predictionsMapStitch = getPredictionResponsesAllClusters(request) val scoredCandidatesStitch = Stitch.traverse(nonCachedSelectedCandidates) { candidate => val tweetId = candidate.candidateIdLong val authorId = candidate.features.getOrElse(AuthorIdFeature, None) val sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None) match { case Some(sourceTweetId) => sourceTweetId case _ => tweetId } predictionsMapStitch.map { predictionsMap => val phoenixPredictionScores = predictionsMap.flatMap { case (phoenixCluster, scoresMapPerTweet) => val actionPredictionsMap = scoresMapPerTweet.getOrElse(sourceTweetId, Map.empty) PhoenixPredictedScoreFeatures.map { feature => val predictions = feature.actions.flatMap(actionPredictionsMap.get) val score = if (predictions.nonEmpty) Some(predictions.max) else None val featureString = f"phoenix.$phoenixCluster.${feature.featureName}" PredictionScore(Some(featureString), score) } }.toSeq val prodScores = PredictedScoreFeatures.map { feature => PredictionScore(Some(feature.statName), candidate.features.getOrElse(feature, None)) } ScoredCandidate( tweetId = tweetId, authorId = authorId, viewerId = query.getOptionalUserId, sourceTweetId = Some(sourceTweetId), servedRequestId = query.features.flatMap(_.getOrElse(PredictionRequestIdFeature, None)), requestTimeMs = Some(query.queryTime.inMillis), predictionScores = Some((phoenixPredictionScores ++ prodScores).toSet) ) } } scoredCandidatesStitch.map { predictionScoresSeq => predictionScoresSeq.map { new ProducerRecord(kafkaTopic, new UnKeyed, _) } } } override def apply( inputs: PipelineResultSideEffect.Inputs[PipelineQuery, HasMarshalling] ): Stitch[Unit] = { val recordsStitch = buildRecordsStitch( query = inputs.query, selectedCandidates = inputs.selectedCandidates, ) recordsStitch.flatMap { records => Stitch .traverse(records) { record => Stitch.callFuture(kafkaProducer.send(record)) } }.unit } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredStatsSideEffect.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.side_effect import com.twitter.core_workflows.user_model.thriftscala.UserState import com.twitter.finagle.stats.Counter import com.twitter.finagle.stats.NullStatsReceiver.NullCounter import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.GrokAnnotationsFeature import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.LowSignalUserFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.ServedAuthorIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SlopAuthorScoreFeature import com.twitter.home_mixer.model.HomeFeatures.TweetMediaIdsFeature import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature import com.twitter.home_mixer.model.PredictedScoreFeature import com.twitter.home_mixer.model.PredictedScoreFeature.PredictedScoreFeatures import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.AuthorListForDataCollectionParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.RequestNormalizedScoresParam import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.CachedScoredTweetsCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsTweetMixerCandidatePipelineConfig import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweets import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines import com.twitter.product_mixer.core.model.common.presentation.CandidateSourcePosition import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch import com.twitter.util.Memoize import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton @Singleton case class ScoredStatsSideEffect @Inject() ( statsReceiver: StatsReceiver) extends PipelineResultSideEffect[ScoredTweetsQuery, ScoredTweetsResponse] with Logging { import ScoredStatsSideEffect._ override val identifier: SideEffectIdentifier = SideEffectIdentifier("ScoredStats") private val baseStatsReceiver = statsReceiver.scope(identifier.toString) private val authorStatsReceiver = baseStatsReceiver.scope("Author") private val authorScoreStatsReceiver = baseStatsReceiver.scope("AuthorScore10000x") private val servedTypeStatsReceiver = baseStatsReceiver.scope("ServedType") private val contentBalanceStatsReceiver = baseStatsReceiver.scope("ContentBalance") private val grokAnnotationsStatsReceiver = baseStatsReceiver.scope("GrokAnnotations") private val inNetworkStatsReceiver = contentBalanceStatsReceiver.scope("InNetwork") private val outOfNetworkStatsReceiver = contentBalanceStatsReceiver.scope("OutOfNetwork") private val replyStatsReceiver = contentBalanceStatsReceiver.scope("Reply") private val originalStatsReceiver = contentBalanceStatsReceiver.scope("Original") private val retweetStatsReceiver = contentBalanceStatsReceiver.scope("Retweet") private val tweetMixerRecallStatsReceiver = baseStatsReceiver.scope("TweetMixerRecall10000x") private val deepRetrievalRecallStatsReceiver = baseStatsReceiver.scope("DeepRetrievalRecall10000x") private val tweetMixerTweetsLandOnTopKStatsReceiver = baseStatsReceiver.scope("TweetMixerTweets") private val deepRetrievalTweetsLandOnTopKStatsReceiver = baseStatsReceiver.scope("DeepRetrievalTweets") private val grokAnnotationsInStatsReceiver = grokAnnotationsStatsReceiver.scope("InNetwork") private val grokAnnotationsOonStatsReceiver = grokAnnotationsStatsReceiver.scope("OutOfNetwork") private val grokAnnotationsReplyStatsReceiver = grokAnnotationsStatsReceiver.scope("Reply") private val signalStatsReceiver = baseStatsReceiver.scope("signal") private val authorDiversityStatsReceiver = baseStatsReceiver.scope("authorDiversity") private val impressedAuthorSizeStat = authorDiversityStatsReceiver.stat("impressedAuthorSize") private val meanImpressedAuthorCountStat = authorDiversityStatsReceiver.stat("meanImpressedAuthorCount1000x") private val grokSlopScoreStatsReceiver = baseStatsReceiver.scope("GrokSlopScore") private val inNetAuthorStatsReceiver = baseStatsReceiver.scope("InNetAuthor") private val StatsReadabilityMultiplier = 1000 private val SmallFollowGraphSize = 25 private val StatsEligibleGrokTopics = Set( "News", "Sports", "Entertainment", "Business & Finance", "Technology", "Politics", "Fashion & Beauty", "Gaming", "Lifestyle", "Movies & TV", "Music", "Travel", "Food", "Comedy", "Health & Fitness" ) // Group of stats to collect per model case class ModelStats(normalized: Boolean, rankingMode: String) { val normalizedScope = if (normalized) "normalized" else "weighted" val scopedStatsReceiver = baseStatsReceiver.scope(normalizedScope).scope(rankingMode) val PredictedScoreCounterName = f"predictedScore${StatsReadabilityMultiplier}xSum" val requestCounter = scopedStatsReceiver.counter("candidatesSum") private val predictedScoreCounters: Map[PredictedScoreFeature, Counter] = PredictedScoreFeatures.map { scoreFeature => ( scoreFeature, scopedStatsReceiver.counter(scoreFeature.statName, PredictedScoreCounterName)) }.toMap def getPredictedScoreCounter(scoreFeature: PredictedScoreFeature): Counter = predictedScoreCounters.getOrElse(scoreFeature, NullCounter) } // Memoize stats object so they are not created per request private val statsPerModel = Memoize[(Boolean, String), ModelStats] { case (normalized, rankingMode) => ModelStats(normalized, rankingMode) } override def apply( inputs: PipelineResultSideEffect.Inputs[ScoredTweetsQuery, ScoredTweetsResponse] ): Stitch[Unit] = { val scopeCandidateMap = Map( "selected" -> inputs.selectedCandidates, "dropped" -> inputs.droppedCandidates, "remaining" -> inputs.remainingCandidates ) // We choose 50 because we don't have per head scores for cached candidates val topSelectedCandidates = inputs.selectedCandidates.take(50) val impressedTweetIds = inputs.query.features.map(_.getOrElse(ImpressedTweets, Seq.empty)).getOrElse(Seq.empty).toSet val servedAuthorMap: Map[Long, Seq[Long]] = inputs.query.features .map(_.getOrElse(ServedAuthorIdsFeature, Map.empty[Long, Seq[Long]])) .getOrElse(Map.empty[Long, Seq[Long]]) recordAuthorDiversityStats(impressedTweetIds, servedAuthorMap) recordScoreStats( topSelectedCandidates, inputs.query.params(RequestNormalizedScoresParam), inputs.query.params(ScoredTweetsParam.TweetMixerRankingModeForStatsRecallAtKParam), inputs.query ) val allCandidates: Seq[CandidateWithDetails] = inputs.selectedCandidates ++ inputs.droppedCandidates recordInNetworkAuthorStats(allCandidates, inputs.query, "retrieved") recordInNetworkAuthorStats(inputs.selectedCandidates, inputs.query, "selected") scopeCandidateMap.map { case (scope, candidates) => recordAuthorStats(candidates, scope, inputs.query.params(AuthorListForDataCollectionParam)) recordServedTypeStats(candidates, scope) recordGrokAnnotationsStats(candidates, scope) recordContentBalanceStats( candidates, scope, inputs.query.params(ScoredTweetsParam.TweetMixerRankingModeForStatsRecallAtKParam) ) recordHasVideoStats(candidates, scope) recordHasMediaStats(candidates, scope) recordSlopScoreStats(candidates, scope) } recordTweetMixerRecallTopKStats( inputs.selectedCandidates.filter( _.source != CachedScoredTweetsCandidatePipelineConfig.Identifier), inputs.droppedCandidates.filter( _.source != CachedScoredTweetsCandidatePipelineConfig.Identifier), inputs.query.params(ScoredTweetsParam.TweetMixerRankingModeForStatsRecallAtKParam) ) recordSignalStats(inputs.query) Stitch.Unit } private def recordAuthorDiversityStats( impressedTweetIds: Set[Long], servedAuthorMap: Map[Long, Seq[Long]] ): Unit = { val authorFreq = servedAuthorMap .collect { case (authorId, tweetIds) => val impressedCount = tweetIds.count(impressedTweetIds.contains) if (impressedCount > 0) Some(authorId -> impressedCount) else None }.flatten.toMap val impressedAuthorSize = authorFreq.size val meanImpressedAuthorCount = if (authorFreq.isEmpty) 0.0f else authorFreq.values.sum.toFloat / impressedAuthorSize impressedAuthorSizeStat.add(impressedAuthorSize) meanImpressedAuthorCountStat.add(meanImpressedAuthorCount * 1000) } private def recordScoreStats( candidates: Seq[CandidateWithDetails], normalized: Boolean, rankingMode: String, query: PipelineQuery ): Unit = { val modelStats = statsPerModel((normalized, rankingMode)) modelStats.requestCounter.incr() PredictedScoreFeatures.map { predictedScoreFeature => val counter = modelStats.getPredictedScoreCounter(predictedScoreFeature) val predictedScoreSum = candidates .map { candidate => predictedScoreFeature .extractScore(candidate.features, query) .getOrElse(0.0) * StatsReadabilityMultiplier }.sum.toInt counter.incr(predictedScoreSum) } } private def recordTweetMixerRecallTopKStats( selectedCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], rankingMode: String ): Unit = { val allCandidates = selectedCandidates ++ droppedCandidates val allCandidatesByHeavyRanker = allCandidates .sortBy( -_.features.getOrElse(WeightedModelScoreFeature, None).getOrElse(Double.NegativeInfinity)) val tweetMixerTweets = allCandidatesByHeavyRanker.filter(fromTweetMixer) val tweetMixerTweetsSortByRank = tweetMixerTweets.sortBy(candidate => candidate.features.get(CandidateSourcePosition)) val deepRetrievalTweets = tweetMixerTweets.filter(tweetMixerTweet => tweetMixerTweet.features.get( ServedTypeFeature) == hmt.ServedType.ForYouDeepRetrieval || tweetMixerTweet.features.get( ServedTypeFeature) == hmt.ServedType.ForYouEvergreenDeepRetrieval) val deepRetrievalTweetsSortByRank = deepRetrievalTweets.sortBy(_.features.get(CandidateSourcePosition)) if (deepRetrievalTweets.nonEmpty) { HeavyRankerTopK.foreach { K => deepRetrievalTweetsLandOnTopKStatsReceiver .scope(rankingMode) .stat(s"LandOnTop${K}") .add(allCandidatesByHeavyRanker .take(K).count { candidate => fromServedType(candidate, hmt.ServedType.ForYouDeepRetrieval) || fromServedType( candidate, hmt.ServedType.ForYouEvergreenDeepRetrieval) }) } deepRetrievalTweetsLandOnTopKStatsReceiver .scope(rankingMode) .stat("LandInSelected") .add(selectedCandidates.count { candidate => fromServedType(candidate, hmt.ServedType.ForYouDeepRetrieval) || fromServedType( candidate, hmt.ServedType.ForYouEvergreenDeepRetrieval) }) DeepRetrievalRecallTopK.foreach { K => val rankedByHeavyRankerScoreTopK = deepRetrievalTweets.take(K) val rankedByTweetMixerRankTopK = deepRetrievalTweetsSortByRank.take(K) val intersectTweets = rankedByHeavyRankerScoreTopK .map(candidate => candidate.candidateIdLong) .intersect(rankedByTweetMixerRankTopK.map(candidate => candidate.candidateIdLong)) deepRetrievalRecallStatsReceiver .scope(rankingMode) .stat(s"RecallAt${K}") .add(intersectTweets.size * 10000 / Math.min(K, deepRetrievalTweets.size)) } } if (tweetMixerTweets.nonEmpty) { TweetMixerRecallTopK.foreach { K => tweetMixerTweetsLandOnTopKStatsReceiver .scope(rankingMode) .stat(s"LandOnTop${K}") .add(allCandidatesByHeavyRanker .take(K).count(fromTweetMixer)) } tweetMixerTweetsLandOnTopKStatsReceiver .scope(rankingMode) .stat("LandInSelected") .add(selectedCandidates.count(fromTweetMixer)) TweetMixerRecallTopK.foreach { K => val rankedByHeavyRankerScoreTopK = tweetMixerTweets.take(K) val rankedByTweetMixerRankTopK = tweetMixerTweetsSortByRank.take(K) val intersectTweets = rankedByHeavyRankerScoreTopK .map(candidate => candidate.candidateIdLong) .intersect(rankedByTweetMixerRankTopK.map(candidate => candidate.candidateIdLong)) tweetMixerRecallStatsReceiver .scope(rankingMode) .stat(s"RecallAt${K}") .add(intersectTweets.size * 10000 / Math.min(K, tweetMixerTweets.size)) } } } def recordAuthorStats( candidates: Seq[CandidateWithDetails], scope: String, authors: Set[Long] ): Unit = { candidates .filter { candidate => candidate.features.getOrElse(AuthorIdFeature, None).exists(authors.contains) && // Only include original tweets (!candidate.features.getOrElse(IsRetweetFeature, false)) && candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty } .groupBy { candidate => val servedType = candidate.features.getOrElse(ServedTypeFeature, hmt.ServedType.Undefined) (servedType.name, candidate.features.get(AuthorIdFeature).get) } .foreach { case ((servedType, authorId), authorCandidates) => val authorStr = authorId.toString.takeRight(9) authorStatsReceiver .scope(scope).scope(authorStr).counter(servedType) .incr(authorCandidates.size) authorCandidates.map { candidate => val score = candidate.features.getOrElse(ScoreFeature, None).getOrElse(0.0).toFloat * 10000 authorScoreStatsReceiver.scope(scope).scope(authorStr).stat(servedType).add(score) } } } def recordServedTypeStats( candidates: Seq[CandidateWithDetails], scope: String ): Unit = { candidates.groupBy(getServedType).foreach { case (servedType, servedTypeCandidates) => servedTypeStatsReceiver.scope(scope).counter(servedType).incr(servedTypeCandidates.size) } } private def recordHasVideoStats( candidates: Seq[CandidateWithDetails], scope: String ): Unit = { candidates.groupBy(_.features.getOrElse(HasVideoFeature, false)).foreach { case (hasVideo, hasVideoCandidates) => servedTypeStatsReceiver .scope(scope).counter(if (hasVideo) "HasVideo" else "NoVideo").incr( hasVideoCandidates.size) } } private def recordHasMediaStats( candidates: Seq[CandidateWithDetails], scope: String ): Unit = { candidates.groupBy(_.features.getOrElse(TweetMediaIdsFeature, Seq[Long]()).nonEmpty).foreach { case (hasMedia, hasMediaCandidates) => servedTypeStatsReceiver .scope(scope).counter(if (hasMedia) "HasMedia" else "NoMedia").incr( hasMediaCandidates.size) } } def recordGrokAnnotationsStats( candidates: Seq[CandidateWithDetails], scope: String ): Unit = { candidates.foreach { candidate => val annotations = candidate.features.getOrElse(GrokAnnotationsFeature, None) val topics = annotations .map(_.topics.filter(StatsEligibleGrokTopics.contains(_))) .filter(_.nonEmpty) val in = candidate.features.getOrElse(InNetworkFeature, true) val reply = candidate.features.getOrElse(InReplyToTweetIdFeature, None).isDefined val baseStat = if (reply) grokAnnotationsReplyStatsReceiver else if (in) grokAnnotationsInStatsReceiver else grokAnnotationsOonStatsReceiver if (topics.isDefined) baseStat.scope(scope).counter("Available").incr() else baseStat.scope(scope).counter("Unavailable").incr() topics.map(_.map(topic => baseStat.counter(topic).incr())) } } def recordSignalStats( query: ScoredTweetsQuery ): Unit = { val userState = query.features.flatMap(_.getOrElse(UserStateFeature, None)) val userStateStr = userState .map { case UserState.New => "new" case UserState.NearZero => "nearZero" case UserState.VeryLight => "veryLight" case UserState.Light => "light" case UserState.MediumTweeter => "medium" case UserState.MediumNonTweeter => "medium" case UserState.HeavyNonTweeter => "heavy" case UserState.HeavyTweeter => "heavy" case UserState.EnumUnknownUserState(_) => "none" }.getOrElse("none") val followGraphSize = query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty).size) val lowFollow = followGraphSize.exists(_ < SmallFollowGraphSize) val lowSignalUser = query.features.map(_.getOrElse(LowSignalUserFeature, false)).getOrElse(false) val stats = signalStatsReceiver.scope(userStateStr) if (lowSignalUser) { if (lowFollow) stats.counter("lowSignal-lowFollow").incr() else stats.counter("lowSignal-okFollow").incr() } else { if (lowFollow) stats.counter("okSignal-lowFollow").incr() else stats.counter("okSignal-okFollow").incr() } } def recordContentBalanceStats( candidates: Seq[CandidateWithDetails], scope: String, rankingMode: String ): Unit = { // Split by network type val (in, oon) = candidates.partition(_.features.getOrElse(InNetworkFeature, true)) inNetworkStatsReceiver.counter(scope).incr(in.size) outOfNetworkStatsReceiver.counter(scope).incr(oon.size) // Track tweet types for all candidates (maintaining backward compatibility) val (reply, nonReply) = candidates.partition(_.features.getOrElse(InReplyToTweetIdFeature, None).isDefined) replyStatsReceiver.counter(scope).incr(reply.size) replyStatsReceiver.scope(rankingMode).counter(scope).incr(reply.size) originalStatsReceiver.scope(rankingMode).counter(scope).incr(nonReply.size) originalStatsReceiver.counter(scope).incr(nonReply.size) // Enhanced tracking: retweets, replies, and original posts for both in-network and out-of-network val (retweets, nonRetweets) = candidates.partition(_.features.getOrElse(IsRetweetFeature, false)) val (replies, originalPosts) = nonRetweets.partition(_.features.getOrElse(InReplyToTweetIdFeature, None).isDefined) // Record retweet stats retweetStatsReceiver.counter(scope).incr(retweets.size) retweetStatsReceiver.scope(rankingMode).counter(scope).incr(retweets.size) // Track by network type val (inRetweets, oonRetweets) = retweets.partition(_.features.getOrElse(InNetworkFeature, true)) val (inReplies, oonReplies) = replies.partition(_.features.getOrElse(InNetworkFeature, true)) val (inOriginalPosts, oonOriginalPosts) = originalPosts.partition(_.features.getOrElse(InNetworkFeature, true)) // In-network tweet type stats retweetStatsReceiver.scope("InNetwork").counter(scope).incr(inRetweets.size) retweetStatsReceiver.scope("InNetwork").scope(rankingMode).counter(scope).incr(inRetweets.size) replyStatsReceiver.scope("InNetwork").counter(scope).incr(inReplies.size) replyStatsReceiver.scope("InNetwork").scope(rankingMode).counter(scope).incr(inReplies.size) originalStatsReceiver.scope("InNetwork").counter(scope).incr(inOriginalPosts.size) originalStatsReceiver .scope("InNetwork").scope(rankingMode).counter(scope).incr(inOriginalPosts.size) // Out-of-network tweet type stats retweetStatsReceiver.scope("OutOfNetwork").counter(scope).incr(oonRetweets.size) retweetStatsReceiver .scope("OutOfNetwork").scope(rankingMode).counter(scope).incr(oonRetweets.size) replyStatsReceiver.scope("OutOfNetwork").counter(scope).incr(oonReplies.size) replyStatsReceiver.scope("OutOfNetwork").scope(rankingMode).counter(scope).incr(oonReplies.size) originalStatsReceiver.scope("OutOfNetwork").counter(scope).incr(oonOriginalPosts.size) originalStatsReceiver .scope("OutOfNetwork").scope(rankingMode).counter(scope).incr(oonOriginalPosts.size) } private def getServedType(candidate: CandidateWithDetails): String = candidate.features.get(ServedTypeFeature).name private def fromTweetMixer(candidate: CandidateWithDetails): Boolean = { candidate.features .get(CandidatePipelines) .contains(ScoredTweetsTweetMixerCandidatePipelineConfig.Identifier) } private def fromServedType( candidate: CandidateWithDetails, servedType: hmt.ServedType ): Boolean = candidate.features.get(ServedTypeFeature) == servedType private def recordSlopScoreStats( candidates: Seq[CandidateWithDetails], scope: String ): Unit = { val count1 = candidates.count(_.features.getOrElse(SlopAuthorScoreFeature, None).contains(1)) val count2 = candidates.count(_.features.getOrElse(SlopAuthorScoreFeature, None).contains(2)) val count3 = candidates.count(_.features.getOrElse(SlopAuthorScoreFeature, None).contains(3)) val noneScore = candidates.count(_.features.getOrElse(SlopAuthorScoreFeature, None).isEmpty) val total = candidates.size grokSlopScoreStatsReceiver.scope(scope).counter("count1").incr(count1) grokSlopScoreStatsReceiver.scope(scope).counter("count2").incr(count2) grokSlopScoreStatsReceiver.scope(scope).counter("count3").incr(count3) grokSlopScoreStatsReceiver.scope(scope).counter("none").incr(noneScore) grokSlopScoreStatsReceiver.scope(scope).counter("total").incr(total) } def recordInNetworkAuthorStats( candidates: Seq[CandidateWithDetails], query: ScoredTweetsQuery, scope: String ): Unit = { val inNet = candidates.filter { candidate => candidate.features.getOrElse(InNetworkFeature, true) && (candidate.features.getOrElse(FromInNetworkSourceFeature, false)) } val total = inNet.size if (total == 0) return val authorCounts: Seq[Int] = inNet .flatMap(_.features.getOrElse(AuthorIdFeature, None)) .groupBy(identity).map { case (k, v) => (k, v.size) }.values.toSeq val followGraphSize = query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty).size).getOrElse(0) val sr = inNetAuthorStatsReceiver.scope(scope) if (followGraphSize > 0) { val recall1000xSGS = (authorCounts.size * 1000) / followGraphSize sr.stat("Recall1000xSGS").add(recall1000xSGS) } val sorted = authorCounts.sorted(Ordering[Int].reverse) def share(k: Int): Int = (sorted.take(k).sum * 1000) / total sr.stat("Top1Share1000x").add(share(1)) sr.stat("Top3Share1000x").add(share(3)) sr.stat("Top10Share1000x").add(share(10)) } override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()) } object ScoredStatsSideEffect { val TweetMixerRecallTopK = Seq(25, 50, 100, 200, 400) val DeepRetrievalRecallTopK = Seq(50, 100, 200) val HeavyRankerTopK = Seq(1, 5, 10, 25, 50, 100, 200, 400) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScoredTweetsDiversityStatsSideEffect.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.side_effect import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.functional_component.feature_hydrator.SimClustersLogFavBasedTweetFeature import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.simclusters_features.SimclustersFeaturesAdapter.SimclustersSparseTweetEmbeddingsFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.CategoryDiversityRescoringParam import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.FeatureHydration.CategoryDiversityKParam import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton import com.twitter.ml.api.DataRecord import scala.jdk.CollectionConverters._ @Singleton case class ScoredTweetsDiversityStatsSideEffect @Inject() ( statsReceiver: StatsReceiver) extends PipelineResultSideEffect[ScoredTweetsQuery, ScoredTweetsResponse] { import ScoredTweetsDiversityStatsSideEffect._ override val identifier: SideEffectIdentifier = SideEffectIdentifier("ScoredTweetsDiversityStats") private val baseStatsReceiver = statsReceiver.scope(identifier.toString) private val diversityStatsReceiverMap = diversityTopK.map { k => k.toString -> baseStatsReceiver.scope("numUniqCategories@k=" + k.toString) }.toMap override def apply( inputs: PipelineResultSideEffect.Inputs[ScoredTweetsQuery, ScoredTweetsResponse] ): Stitch[Unit] = { val query = inputs.query val selectedCandidates = inputs.selectedCandidates if (query.params(CategoryDiversityRescoringParam)) { diversityTopK.foreach { topK => getNumUniqueCategoriesStats( query, selectedCandidates, topK ) } } Stitch.Unit } private def getNumUniqueCategoriesStats( query: ScoredTweetsQuery, selectedCandidates: Seq[CandidateWithDetails], topK: Int ): Unit = { val topKSelectedCandidates = selectedCandidates .sortBy(_.features.getOrElse(ScoreFeature, None).getOrElse(0.0)) .take(topK) val categories = topKSelectedCandidates.map { candidate => val topKClustersMap = Option( candidate.features .getOrElse(SimClustersLogFavBasedTweetFeature, new DataRecord()) .getSparseContinuousFeatures ).flatMap(mapOpt => Option(mapOpt.get(SimclustersSparseTweetEmbeddingsFeature.getFeatureId))) topKClustersMap .map(_.asScala.toSeq.sortBy(-_._2).take(query.params(CategoryDiversityKParam)).map(_._1)) .getOrElse(Seq.empty) } // assign the num of unique categories to the stats receiver val uniqueCategories = categories.flatten.toSet diversityStatsReceiverMap(topK.toString).stat("numUniqCategories").add(uniqueCategories.size) } } object ScoredTweetsDiversityStatsSideEffect { private val diversityTopK = Array(5, 15, 30) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeScoredCandidatesSideEffect.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.side_effect import com.twitter.finagle.tracing.Trace import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature import com.twitter.home_mixer.model.HomeFeatures.GrokIsGoreFeature import com.twitter.home_mixer.model.HomeFeatures.GrokIsLowQualityFeature import com.twitter.home_mixer.model.HomeFeatures.GrokIsNsfwFeature import com.twitter.home_mixer.model.HomeFeatures.GrokIsOcrFeature import com.twitter.home_mixer.model.HomeFeatures.GrokIsSpamFeature import com.twitter.home_mixer.model.HomeFeatures.GrokIsViolentFeature import com.twitter.home_mixer.model.HomeFeatures.GrokTopCategoryFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SourceSignalFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.TweetMixerScoreFeature import com.twitter.home_mixer.model.PredictedScoreFeature import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeScoredCandidatesFlag import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableScribeScoredCandidatesParam import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.side_effect.ScribeLogEventSideEffect import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidatePipelines import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures import com.twitter.timelines.timeline_logging.{thriftscala => t} import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Singleton /** * Side effect that logs scored candidates from scoring pipelines */ @Singleton class ScribeScoredCandidatesSideEffect @Inject() ( @Flag(ScribeScoredCandidatesFlag) enableScribeScoredCandidates: Boolean, eventBusPublisher: EventPublisher[t.ScoredCandidate]) extends ScribeLogEventSideEffect[ t.ScoredCandidate, ScoredTweetsQuery, ScoredTweetsResponse ] with PipelineResultSideEffect.Conditionally[ ScoredTweetsQuery, ScoredTweetsResponse ] with Logging { override val identifier: SideEffectIdentifier = SideEffectIdentifier("ScribeScoredCandidates") override def onlyIf( query: ScoredTweetsQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: ScoredTweetsResponse ): Boolean = enableScribeScoredCandidates && query.params(EnableScribeScoredCandidatesParam) /** * Build the log events from query, selections and response * * @param query PipelineQuery * @param selectedCandidates Result after Selectors are executed * @param remainingCandidates Candidates which were not selected * @param droppedCandidates Candidates dropped during selection * @param response Result after Unmarshalling * * @return LogEvent in thrift */ override def buildLogEvents( query: ScoredTweetsQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: ScoredTweetsResponse ): Seq[t.ScoredCandidate] = { val returned = (selectedCandidates ++ remainingCandidates).map(toThrift(_, query, false)) val dropped = droppedCandidates.map(toThrift(_, query, true)) returned ++ dropped } private def toThrift( candidate: CandidateWithDetails, query: ScoredTweetsQuery, isDropped: Boolean ): t.ScoredCandidate = { val grokMetadata = t.GrokMetadata( isGore = candidate.features.getOrElse(GrokIsGoreFeature, None), isNsfw = candidate.features.getOrElse(GrokIsNsfwFeature, None), isSpam = candidate.features.getOrElse(GrokIsSpamFeature, None), isViolent = candidate.features.getOrElse(GrokIsViolentFeature, None), isLowQuality = candidate.features.getOrElse(GrokIsLowQualityFeature, None), isOcr = candidate.features.getOrElse(GrokIsOcrFeature, None) ) t.ScoredCandidate( tweetId = candidate.candidateIdLong, viewerId = query.getOptionalUserId, authorId = candidate.features.getOrElse(AuthorIdFeature, None), traceId = Some(Trace.id.traceId.toLong), requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None)), score = candidate.features.getOrElse(ScoreFeature, None), suggestType = Some(candidate.features.get(ServedTypeFeature).name), isInNetwork = candidate.features.getTry(FromInNetworkSourceFeature).toOption, inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None), inReplyToUserId = candidate.features.getOrElse(InReplyToUserIdFeature, None), quotedTweetId = candidate.features.getOrElse(QuotedTweetIdFeature, None), quotedUserId = candidate.features.getOrElse(QuotedUserIdFeature, None), directedAtUserId = candidate.features.getOrElse(DirectedAtUserIdFeature, None), favoritedByUserIds = convertSeqFeature(candidate, FavoritedByUserIdsFeature), followedByUserIds = convertSeqFeature(candidate, FollowedByUserIdsFeature), ancestors = convertSeqFeature(candidate, AncestorsFeature), requestTimeMs = Some(query.queryTime.inMilliseconds), candidatePipelineIdentifier = candidate.features.getTry(CandidatePipelines).toOption.map(_.head.name), earlybirdScore = candidate.features.getOrElse(EarlybirdScoreFeature, None), isDropped = Some(isDropped), servedRequestId = query.features.flatMap(_.getOrElse(ServedIdFeature, None)), sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), predictionScores = Some(extractPredictionScores(candidate) + extractCandidateSourceScore(candidate)), grokAnnotation = Some( t.GrokAnnotation( category = candidate.features .getOrElse(GrokTopCategoryFeature, None), grokMetadata = Some(grokMetadata) )), sourceSignal = candidate.features.getOrElse(SourceSignalFeature, None).map { signal => t.SourceSignal( id = Some(signal.id), signalType = signal.signalType, signalEntity = signal.signalEntity, authorId = signal.authorId, ) } ) } private def extractPredictionScores(candidate: CandidateWithDetails): Set[t.PredictionScore] = PredictedScoreFeature.PredictedScoreFeatureSet .map(feature => t.PredictionScore(Some(feature.featureName), candidate.features.getOrElse(feature, None))) .filter(_.score.isDefined) private def extractCandidateSourceScore(candidate: CandidateWithDetails): t.PredictionScore = t.PredictionScore( Some(TimelinesSharedFeatures.CANDIDATE_SOURCE_SCORE.getFeatureName), candidate.features .getOrElse(TweetMixerScoreFeature, None)) private def convertSeqFeature[T]( candidateWithDetails: CandidateWithDetails, feature: Feature[_, Seq[T]] ): Option[Seq[T]] = Option( candidateWithDetails.features .getOrElse(feature, Seq.empty)).filter(_.nonEmpty) override val logPipelinePublisher: EventPublisher[t.ScoredCandidate] = eventBusPublisher override val alerts = Seq(HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect/ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.side_effect import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mysql.Client import com.twitter.finagle.mysql.Transactions import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.util.DefaultTimer import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.param.HomeMixerFlagName.DataRecordMetadataStoreConfigsYmlFlag import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCommonFeaturesAndCandidateFeaturesFlag import com.twitter.home_mixer.param.HomeMixerInjectionNames.CandidateFeaturesScribeEventPublisher import com.twitter.home_mixer.param.HomeMixerInjectionNames.CommonFeaturesScribeEventPublisher import com.twitter.home_mixer.param.HomeMixerInjectionNames.MinimumFeaturesScribeEventPublisher import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features.NonMLCandidateFeatures import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features.NonMLCandidateFeaturesAdapter import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features.NonMLCommonFeatures import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features.NonMLCommonFeaturesAdapter import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse import com.twitter.home_mixer.product.scored_tweets.scorer.CandidateFeaturesDataRecordFeature import com.twitter.home_mixer.product.scored_tweets.scorer.CommonFeaturesDataRecordFeature import com.twitter.home_mixer.product.scored_tweets.scorer.PredictedScoreFeature.PredictedScoreFeatures import com.twitter.home_mixer.util.CandidatesUtil.getOriginalAuthorId import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.ml.api.DataRecordMerger import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter import com.twitter.product_mixer.core.feature.featuremap.datarecord.SpecificFeatures import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.stitch.Stitch import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.CandidateAndCommonFeaturesStreamingUtils import com.twitter.timelines.ml.pldr.client.MysqlClientUtils import com.twitter.timelines.ml.pldr.client.VersionedMetadataCacheClient import com.twitter.timelines.ml.pldr.conversion.VersionIdAndFeatures import com.twitter.timelines.suggests.common.data_record_metadata.{thriftscala => drmd} import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} import com.twitter.timelines.util.stats.OptionObserver import com.twitter.util.Time import com.twitter.util.Try import com.twitter.util.logging.Logging import javax.inject.Inject import javax.inject.Named import javax.inject.Singleton import scala.collection.JavaConverters._ /** * (1) Scribe common features sent to prediction service + some other features as PLDR format into logs * (2) Scribe candidate features sent to prediction service + some other features as PLDR format into another logs */ @Singleton class ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect @Inject() ( @Flag(DataRecordMetadataStoreConfigsYmlFlag) dataRecordMetadataStoreConfigsYml: String, @Flag(ScribeServedCommonFeaturesAndCandidateFeaturesFlag) enableScribeServedCommonFeaturesAndCandidateFeatures: Boolean, @Named(CommonFeaturesScribeEventPublisher) commonFeaturesScribeEventPublisher: EventPublisher[ pldr.PolyDataRecord ], @Named(CandidateFeaturesScribeEventPublisher) candidateFeaturesScribeEventPublisher: EventPublisher[ pldr.PolyDataRecord ], @Named(MinimumFeaturesScribeEventPublisher) minimumFeaturesScribeEventPublisher: EventPublisher[ pldr.PolyDataRecord ], statsReceiver: StatsReceiver, ) extends PipelineResultSideEffect[ScoredTweetsQuery, ScoredTweetsResponse] with PipelineResultSideEffect.Conditionally[ScoredTweetsQuery, ScoredTweetsResponse] with Logging { override val identifier: SideEffectIdentifier = SideEffectIdentifier("ScribeServedCommonFeaturesAndCandidateFeatures") private val drMerger = new DataRecordMerger private val postScoringCandidateFeatures = SpecificFeatures(PredictedScoreFeatures) private val postScoringCandidateFeaturesDataRecordAdapter = new DataRecordConverter(postScoringCandidateFeatures) private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) private val metadataFetchFailedCounter = scopedStatsReceiver.counter("metadataFetchFailed") private val commonFeaturesScribeCounter = scopedStatsReceiver.counter("commonFeaturesScribe") private val commonFeaturesPLDROptionObserver = OptionObserver(scopedStatsReceiver.scope("commonFeaturesPLDR")) private val candidateFeaturesScribeCounter = scopedStatsReceiver.counter("candidateFeaturesScribe") private val candidateFeaturesPLDROptionObserver = OptionObserver(scopedStatsReceiver.scope("candidateFeaturesPLDR")) private val minimumFeaturesPLDROptionObserver = OptionObserver(scopedStatsReceiver.scope("minimumFeaturesPLDR")) private val minimumFeaturesScribeCounter = scopedStatsReceiver.counter("minimumFeaturesScribe") lazy private val dataRecordMetadataStoreClient: Option[Client with Transactions] = Try { MysqlClientUtils.mysqlClientProvider( MysqlClientUtils.parseConfigFromYaml(dataRecordMetadataStoreConfigsYml)) }.onFailure { e => info(s"Error building MySQL client: $e") }.toOption lazy private val versionedMetadataCacheClientOpt: Option[ VersionedMetadataCacheClient[Map[drmd.FeaturesCategory, Option[VersionIdAndFeatures]]] ] = dataRecordMetadataStoreClient.map { mysqlClient => new VersionedMetadataCacheClient[Map[drmd.FeaturesCategory, Option[VersionIdAndFeatures]]]( maximumSize = 1, expireDurationOpt = None, mysqlClient = mysqlClient, transform = CandidateAndCommonFeaturesStreamingUtils.metadataTransformer, statsReceiver = statsReceiver ) } versionedMetadataCacheClientOpt.foreach { versionedMetadataCacheClient => versionedMetadataCacheClient .metadataFetchTimerTask( CandidateAndCommonFeaturesStreamingUtils.metadataFetchKey, metadataFetchTimer = DefaultTimer, metadataFetchInterval = 90.seconds, metadataFetchFailedCounter = metadataFetchFailedCounter ) } override def onlyIf( query: ScoredTweetsQuery, selectedCandidates: Seq[CandidateWithDetails], remainingCandidates: Seq[CandidateWithDetails], droppedCandidates: Seq[CandidateWithDetails], response: ScoredTweetsResponse ): Boolean = enableScribeServedCommonFeaturesAndCandidateFeatures override def apply( inputs: PipelineResultSideEffect.Inputs[ScoredTweetsQuery, ScoredTweetsResponse] ): Stitch[Unit] = { Stitch.value { val servedTimestamp: Long = Time.now.inMilliseconds val nonMLCommonFeatures = NonMLCommonFeatures( userId = inputs.query.getRequiredUserId, predictionRequestId = inputs.query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)), servedTimestamp = servedTimestamp ) val nonMLCommonFeaturesDataRecord = NonMLCommonFeaturesAdapter.adaptToDataRecords(nonMLCommonFeatures).asScala.head /** * Steps of scribing common features * (1) fetch common features as data record * (2) extract additional feature as data record, e.g. predictionRequestId which is used as join key in downstream jobs * (3) merge two data records above and convert the merged data record to pldr * (4) publish pldr */ val commonFeaturesDataRecordOpt = inputs.selectedCandidates.headOption.map(_.features.get(CommonFeaturesDataRecordFeature)) val commonFeaturesPLDROpt = commonFeaturesDataRecordOpt.flatMap { commonFeaturesDataRecord => drMerger.merge(commonFeaturesDataRecord, nonMLCommonFeaturesDataRecord) CandidateAndCommonFeaturesStreamingUtils.commonFeaturesToPolyDataRecord( versionedMetadataCacheClientOpt = versionedMetadataCacheClientOpt, commonFeatures = commonFeaturesDataRecord, valueFormat = pldr.PolyDataRecord._Fields.LITE_COMPACT_DATA_RECORD ) } commonFeaturesPLDROptionObserver(commonFeaturesPLDROpt).foreach { pldr => commonFeaturesScribeEventPublisher.publish(pldr) commonFeaturesScribeCounter.incr() } /** * steps of scribing candidate features * (1) fetch candidate features as data record * (2) extract additional features (mostly non ML features including predicted scores, predictionRequestId, userId, tweetId) * (3) merge data records and convert the merged data record into pldr * (4) publish pldr */ inputs.selectedCandidates.foreach { candidate => val candidateFeaturesDataRecord = candidate.features.get(CandidateFeaturesDataRecordFeature) /** * extract predicted scores as data record and merge it into original data record */ val postScoringCandidateFeaturesDataRecord = postScoringCandidateFeaturesDataRecordAdapter.toDataRecord(candidate.features) drMerger.merge(candidateFeaturesDataRecord, postScoringCandidateFeaturesDataRecord) /** * extract non ML common features as data record and merge it into original data record */ drMerger.merge(candidateFeaturesDataRecord, nonMLCommonFeaturesDataRecord) /** * extract non ML candidate features as data record and merge it into original data record */ val nonMLCandidateFeatures = NonMLCandidateFeatures( tweetId = candidate.candidateIdLong, sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None), originalAuthorId = getOriginalAuthorId(candidate.features) ) val nonMLCandidateFeaturesDataRecord = NonMLCandidateFeaturesAdapter.adaptToDataRecords(nonMLCandidateFeatures).asScala.head drMerger.merge(candidateFeaturesDataRecord, nonMLCandidateFeaturesDataRecord) val candidateFeaturesPLDROpt = CandidateAndCommonFeaturesStreamingUtils.candidateFeaturesToPolyDataRecord( versionedMetadataCacheClientOpt = versionedMetadataCacheClientOpt, candidateFeatures = candidateFeaturesDataRecord, valueFormat = pldr.PolyDataRecord._Fields.LITE_COMPACT_DATA_RECORD ) candidateFeaturesPLDROptionObserver(candidateFeaturesPLDROpt).foreach { pldr => candidateFeaturesScribeEventPublisher.publish(pldr) candidateFeaturesScribeCounter.incr() } // scribe minimum features which are used to join labels from client events. val minimumFeaturesPLDROpt = candidateFeaturesPLDROpt .map(CandidateAndCommonFeaturesStreamingUtils.extractMinimumFeaturesFromPldr) .map(pldr.PolyDataRecord.dataRecord) minimumFeaturesPLDROptionObserver(minimumFeaturesPLDROpt).foreach { pldr => minimumFeaturesScribeEventPublisher.publish(pldr) minimumFeaturesScribeCounter.incr() } } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/util/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ "3rdparty/jvm/com/github/nscala_time", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/embedding", "snowflake/src/main/scala/com/twitter/snowflake/id", "src/thrift/com/twitter/timelines/control_ai:timeline-control-ai-thrift-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/util/ControlAiUtil.scala ================================================ package com.twitter.home_mixer.product.scored_tweets.util import com.github.nscala_time.time.Imports.LocalDate import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.product_mixer.component_library.feature_hydrator.candidate.embedding.TweetTextV8EmbeddingFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.snowflake.id.SnowflakeId import com.twitter.timelines.control_ai.control.{thriftscala => ci} object ControlAiUtil { private def isDotProductClose( vector1: Option[Seq[Double]], vector2: Option[Seq[Double]], threshold: Double ): Boolean = { (vector1, vector2) match { case (Some(v1), Some(v2)) => require(v1.length == v2.length) val dotProduct = v1.zip(v2).map { case (a, b) => a * b }.sum dotProduct > threshold case _ => false } } def conditionMatch( action: ci.Action, candidate: CandidateWithFeatures[TweetCandidate], topicMap: Map[String, Seq[Double]], threshold: Double ): Boolean = { val now = LocalDate.now() val currentTime = now.toDate.toInstant.getEpochSecond val currentDayOfWeek = now.getDayOfWeek val currentHourOfDay = now.toDateTimeAtCurrentTime.toDateTime.getHourOfDay val conditions: Seq[ci.Condition => Boolean] = Seq( _.postTopic.forall { topic => isDotProductClose( topicMap.get(topic), candidate.features.getOrElse(TweetTextV8EmbeddingFeature, None), threshold) }, _.postLanguage.forall(candidate.features.getOrElse(TweetLanguageFeature, None).contains), _.postHasVideo.forall(_ == candidate.features.getOrElse(HasVideoFeature, false)), _.postHasImage.forall(_ == candidate.features.getOrElse(HasImageFeature, false)), _.postIsReply.forall( _ == candidate.features.getOrElse(InReplyToTweetIdFeature, None).isDefined), _.postIsRetweet.forall(_ == candidate.features.getOrElse(IsRetweetFeature, false)), _.postMaximumAge.forall(maxAge => SnowflakeId.timeFromIdOpt(candidate.candidate.id).exists(_.untilNow.inMinutes <= maxAge)), _.userFollowsAuthor.forall(_ == candidate.features.get(InNetworkFeature)), _.postLikesCountGreaterThan.forall(count => candidate.features .getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= count))), _.postLikesCountLessThan.forall(count => candidate.features .getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ < count))), _.postRetweetsCountGreaterThan.forall(count => candidate.features .getOrElse(EarlybirdFeature, None).exists(_.retweetCountV2.exists(_ >= count))), _.postRetweetsCountLessThan.forall(count => candidate.features .getOrElse(EarlybirdFeature, None).exists(_.retweetCountV2.exists(_ < count))), _.postRepliesCountGreaterThan.forall(count => candidate.features .getOrElse(EarlybirdFeature, None).exists(_.replyCountV2.exists(_ >= count))), _.postRepliesCountLessThan.forall(count => candidate.features .getOrElse(EarlybirdFeature, None).exists(_.replyCountV2.exists(_ < count))), _.postQuotesCountGreaterThan.forall(count => candidate.features .getOrElse(EarlybirdFeature, None).exists(_.quoteCount.exists(_ >= count))), _.postQuotesCountLessThan.forall(count => candidate.features .getOrElse(EarlybirdFeature, None).exists(_.quoteCount.exists(_ < count))), _.postLikedByFollowings.forall( _ == candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty).nonEmpty), _.authorId.forall(aid => candidate.features.getOrElse(AuthorIdFeature, None).contains(aid)), _.authorLanguage.forall(lang => candidate.features.getOrElse(TweetLanguageFeature, None).contains(lang)), _.authorAccountAgeGreaterThan.forall(count => candidate.features.getOrElse(AuthorAccountAge, None).exists(_.inDays / 365 >= count)), _.authorAccountAgeLessThan.forall(count => candidate.features.getOrElse(AuthorAccountAge, None).exists(_.inDays / 365 < count)), _.authorFollowerCountGreaterThan.forall(count => candidate.features.getOrElse(AuthorFollowersFeature, None).exists(_ >= count)), _.authorFollowerCountLessThan.forall(count => candidate.features.getOrElse(AuthorFollowersFeature, None).exists(_ < count)), _.authorFollowedByFollowings.forall( _ == candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty), _.queryValidStartTime.forall(currentTime >= _), _.queryValidEndTime.forall(currentTime < _), _.queryValidDayOfWeek.forall(_.toInt == currentDayOfWeek), _.queryValidHourOfDay.forall(_.toInt == currentHourOfDay) ) conditions.forall(_(action.condition)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/location", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/cursor/timelines", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder/earlybird", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", "src/java/com/twitter/search/common/schema/earlybird", "src/java/com/twitter/search/queryparser/query:core-query-nodes", "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", "src/thrift/com/twitter/search:earlybird-scala", "stitch/stitch-tweetypie", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", ], exports = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "src/thrift/com/twitter/timelines/render:thrift-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdCandidatePipelineConfig.scala ================================================ package com.twitter.home_mixer.product.subscribed import com.google.inject.Inject import com.twitter.home_mixer.product.subscribed.model.SubscribedQuery import com.twitter.product_mixer.component_library.candidate_source.earlybird.EarlybirdTweetCandidateSource import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSSubscribedUsersFeature import com.twitter.product_mixer.component_library.filter.TweetVisibilityFilter import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.search.earlybird.{thriftscala => t} import com.twitter.spam.rtf.thriftscala.SafetyLevel.TimelineHomeSubscribed import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} import com.twitter.tweetypie.thriftscala.TweetVisibilityPolicy class SubscribedEarlybirdCandidatePipelineConfig @Inject() ( earlybirdTweetCandidateSource: EarlybirdTweetCandidateSource, tweetyPieStitchClient: TweetypieStitchClient, subscribedEarlybirdQueryTransformer: SubscribedEarlybirdQueryTransformer) extends CandidatePipelineConfig[ SubscribedQuery, t.EarlybirdRequest, t.ThriftSearchResult, TweetCandidate ] { override val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("SubscribedEarlybird") override val candidateSource: BaseCandidateSource[t.EarlybirdRequest, t.ThriftSearchResult] = earlybirdTweetCandidateSource override val gates: Seq[Gate[SubscribedQuery]] = Seq( NonEmptySeqFeatureGate(SGSSubscribedUsersFeature) ) override def filters: Seq[Filter[SubscribedQuery, TweetCandidate]] = Seq( new TweetVisibilityFilter( tweetyPieStitchClient, TweetVisibilityPolicy.UserVisible, TimelineHomeSubscribed ) ) override val queryTransformer: CandidatePipelineQueryTransformer[ SubscribedQuery, t.EarlybirdRequest ] = subscribedEarlybirdQueryTransformer override val featuresFromCandidateSourceTransformers: Seq[ CandidateFeatureTransformer[t.ThriftSearchResult] ] = Seq(SubscribedEarlybirdResponseFeatureTransformer) override val resultTransformer: CandidatePipelineResultsTransformer[ t.ThriftSearchResult, TweetCandidate ] = { sourceResult => TweetCandidate(id = sourceResult.id) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdQueryTransformer.scala ================================================ package com.twitter.home_mixer.product.subscribed import com.twitter.finagle.thrift.ClientId import com.twitter.finagle.tracing.Trace import com.twitter.home_mixer.product.subscribed.model.SubscribedQuery import com.twitter.home_mixer.product.subscribed.param.SubscribedParam.ServerMaxResultsParam import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSSubscribedUsersFeature import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant import com.twitter.search.earlybird.{thriftscala => t} import com.twitter.search.queryparser.query.Conjunction import com.twitter.search.queryparser.query.search.SearchOperator import javax.inject.Inject import javax.inject.Singleton @Singleton case class SubscribedEarlybirdQueryTransformer @Inject() (clientId: ClientId) extends CandidatePipelineQueryTransformer[SubscribedQuery, t.EarlybirdRequest] { override def transform(query: SubscribedQuery): t.EarlybirdRequest = { val subscribedUserIds = query.features.map(_.get(SGSSubscribedUsersFeature)).getOrElse(Seq.empty) val subscribedUsersQuery = new SearchOperator.Builder() .setType(SearchOperator.Type.FILTER) .addOperand(EarlybirdFieldConstant.EXCLUSIVE_FILTER_TERM) .build() val searchQuery = query.pipelineCursor .map { cursor => val sinceIdQuery = (id: Long) => new SearchOperator(SearchOperator.Type.SINCE_ID, id.toString) val maxIdQuery = // max ID is inclusive, so subtract 1 (id: Long) => new SearchOperator(SearchOperator.Type.MAX_ID, (id - 1).toString) (cursor.cursorType, cursor.id, cursor.gapBoundaryId) match { case (Some(TopCursor), Some(sinceId), _) => new Conjunction(sinceIdQuery(sinceId), subscribedUsersQuery) case (Some(BottomCursor), Some(maxId), _) => new Conjunction(maxIdQuery(maxId), subscribedUsersQuery) case (Some(GapCursor), Some(maxId), Some(sinceId)) => new Conjunction(sinceIdQuery(sinceId), maxIdQuery(maxId), subscribedUsersQuery) case (Some(GapCursor), _, _) => throw PipelineFailure(MalformedCursor, "Invalid cursor " + cursor.toString) case _ => subscribedUsersQuery } }.getOrElse(subscribedUsersQuery) t.EarlybirdRequest( searchQuery = t.ThriftSearchQuery( serializedQuery = Some(searchQuery.serialize), fromUserIDFilter64 = Some(subscribedUserIds), numResults = query.requestedMaxResults.getOrElse(query.params(ServerMaxResultsParam)), rankingMode = t.ThriftSearchRankingMode.Recency, ), getOlderResults = Some(true), // needed for archive access to older tweets clientRequestID = Some(s"${Trace.id.traceId}"), numResultsToReturnAtRoot = Some(query.params(ServerMaxResultsParam)), clientId = Some(clientId.name), ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedEarlybirdResponseFeatureTransformer.scala ================================================ package com.twitter.home_mixer.product.subscribed import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier import com.twitter.search.earlybird.{thriftscala => t} object SubscribedEarlybirdResponseFeatureTransformer extends CandidateFeatureTransformer[t.ThriftSearchResult] { override val identifier: TransformerIdentifier = TransformerIdentifier("SubscribedEarlybirdResponse") override val features: Set[Feature[_, _]] = Set( AuthorIdFeature, InReplyToTweetIdFeature, IsRetweetFeature, SourceTweetIdFeature, SourceUserIdFeature, ) override def transform(candidate: t.ThriftSearchResult): FeatureMap = FeatureMapBuilder() .add(AuthorIdFeature, candidate.tweetypieTweet.flatMap(_.coreData.map(_.userId))) .add( InReplyToTweetIdFeature, candidate.tweetypieTweet.flatMap(_.coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId)))) .add(IsRetweetFeature, candidate.metadata.exists(_.isRetweet.contains(true))) .add(SourceTweetIdFeature, candidate.sourceTweetypieTweet.map(_.id)) .add(SourceUserIdFeature, candidate.sourceTweetypieTweet.flatMap(_.coreData.map(_.userId))) .build() } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedMixerPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.subscribed import com.twitter.clientapp.{thriftscala => ca} import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.home_mixer.candidate_pipeline.ConversationServiceCandidatePipelineConfigBuilder import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig import com.twitter.home_mixer.functional_component.feature_hydrator._ import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration import com.twitter.home_mixer.functional_component.side_effect._ import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.subscribed.model.SubscribedQuery import com.twitter.home_mixer.product.subscribed.param.SubscribedParam.EnablePostContextFeatureHydratorParam import com.twitter.home_mixer.product.subscribed.param.SubscribedParam.ServerMaxResultsParam import com.twitter.home_mixer.util.CandidatesUtil import com.twitter.inject.annotations.Flag import com.twitter.logpipeline.client.common.EventPublisher import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.location.UserLocationQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSSubscribedUsersQueryFeatureHydrator import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AddEntriesWithReplaceAndShowAlertInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedCursorIdSelector.TweetIdSelector import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedGapCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder import com.twitter.product_mixer.component_library.premarshaller.urt.builder.earlybird.EarlybirdGapIncludeInstruction import com.twitter.product_mixer.component_library.selector.DropMaxCandidates import com.twitter.product_mixer.component_library.selector.InsertAppendResults import com.twitter.product_mixer.component_library.selector.SelectConditionally import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller import com.twitter.product_mixer.core.functional_component.selector.Selector import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig import com.twitter.product_mixer.core.pipeline.FailOpenPolicy import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.stringcenter.client.StringCenter import com.twitter.timelines.render.{thriftscala => urt} import javax.inject.Inject import javax.inject.Provider import javax.inject.Singleton @Singleton class SubscribedMixerPipelineConfig @Inject() ( subscribedEarlybirdCandidatePipelineConfig: SubscribedEarlybirdCandidatePipelineConfig, conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[ SubscribedQuery ], editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[SubscribedQuery], dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, requestQueryFeatureHydrator: RequestQueryFeatureHydrator[SubscribedQuery], sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, sgsSubscribedUsersQueryFeatureHydrator: SGSSubscribedUsersQueryFeatureHydrator, memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, userLocationQueryFeatureHydrator: UserLocationQueryFeatureHydrator, publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, homeTimelineServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], externalStrings: HomeMixerExternalStrings, @ProductScoped stringCenterProvider: Provider[StringCenter], urtTransportMarshaller: UrtTransportMarshaller, @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) extends MixerPipelineConfig[SubscribedQuery, Timeline, urt.TimelineResponse] { override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("Subscribed") private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep private val resultSelectorsStep = MixerPipelineConfig.resultSelectorsStep override val fetchQueryFeatures: Seq[QueryFeatureHydrator[SubscribedQuery]] = Seq( requestQueryFeatureHydrator, sgsFollowedUsersQueryFeatureHydrator, sgsSubscribedUsersQueryFeatureHydrator, AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), AsyncQueryFeatureHydrator(dependentCandidatesStep, userLocationQueryFeatureHydrator), AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator) ) private val earlybirdCandidatePipelineScope = SpecificPipeline(subscribedEarlybirdCandidatePipelineConfig.identifier) private val conversationServiceCandidatePipelineConfig = conversationServiceCandidatePipelineConfigBuilder.build( earlybirdCandidatePipelineScope, hmt.ServedType.Subscribed, EnablePostContextFeatureHydratorParam ) override val candidatePipelines: Seq[CandidatePipelineConfig[SubscribedQuery, _, _, _]] = Seq(subscribedEarlybirdCandidatePipelineConfig) override val dependentCandidatePipelines: Seq[ DependentCandidatePipelineConfig[SubscribedQuery, _, _, _] ] = Seq( conversationServiceCandidatePipelineConfig, editedTweetsCandidatePipelineConfig, newTweetsPillCandidatePipelineConfig ) override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, ) override val resultSelectors: Seq[Selector[SubscribedQuery]] = Seq( UpdateSortCandidates( ordering = CandidatesUtil.reverseChronTweetsOrdering, candidatePipeline = conversationServiceCandidatePipelineConfig.identifier ), DropMaxCandidates( candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, maxSelectionsParam = MaxNumberReplaceInstructionsParam ), DropMaxCandidates( candidatePipeline = conversationServiceCandidatePipelineConfig.identifier, maxSelectionsParam = ServerMaxResultsParam ), InsertAppendResults(candidatePipeline = conversationServiceCandidatePipelineConfig.identifier), // This selector must come after the tweets are inserted into the results UpdateNewTweetsPillDecoration( pipelineScope = SpecificPipelines( conversationServiceCandidatePipelineConfig.identifier, newTweetsPillCandidatePipelineConfig.identifier ), stringCenter = stringCenterProvider.get(), seeNewTweetsString = externalStrings.seeNewTweetsString, tweetedString = externalStrings.tweetedString ), InsertAppendResults(candidatePipeline = editedTweetsCandidatePipelineConfig.identifier), SelectConditionally( selector = InsertAppendResults(candidatePipeline = newTweetsPillCandidatePipelineConfig.identifier), includeSelector = (_, _, results) => CandidatesUtil.containsType[TweetCandidate](results) ), UpdateHomeClientEventDetails( candidatePipelines = Set(conversationServiceCandidatePipelineConfig.identifier) ), ) private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( enableScribeClientEvents = enableScribeClientEvents, logPipelinePublisher = clientEventsScribeEventPublisher, injectedTweetsCandidatePipelineIdentifiers = Seq(conversationServiceCandidatePipelineConfig.identifier), ) override val resultSideEffects: Seq[PipelineResultSideEffect[SubscribedQuery, Timeline]] = Seq( homeScribeClientEventSideEffect, homeTimelineServedCandidatesSideEffect, publishClientSentImpressionsEventBusSideEffect ) override val domainMarshaller: DomainMarshaller[SubscribedQuery, Timeline] = { val instructionBuilders = Seq( ReplaceEntryInstructionBuilder(ReplaceAllEntries), AddEntriesWithReplaceAndShowAlertInstructionBuilder(), ShowAlertInstructionBuilder(), ) val topCursorBuilder = OrderedTopCursorBuilder(TweetIdSelector) val bottomCursorBuilder = OrderedBottomCursorBuilder(TweetIdSelector, EarlybirdGapIncludeInstruction.inverse()) val gapCursorBuilder = OrderedGapCursorBuilder(TweetIdSelector, EarlybirdGapIncludeInstruction) val scribeConfigBuilder = StaticTimelineScribeConfigBuilder(TimelineScribeConfig(Some("subscribed"), None, None)) val metadataBuilder = UrtMetadataBuilder(scribeConfigBuilder = Some(scribeConfigBuilder)) UrtDomainMarshaller( instructionBuilders = instructionBuilders, metadataBuilder = Some(metadataBuilder), cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder, gapCursorBuilder) ) } override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = urtTransportMarshaller } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/SubscribedProductPipelineConfig.scala ================================================ package com.twitter.home_mixer.product.subscribed import com.twitter.conversions.DurationOps._ import com.twitter.home_mixer.model.request.HomeMixerRequest import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.home_mixer.model.request.SubscribedProductContext import com.twitter.home_mixer.product.subscribed.model.SubscribedQuery import com.twitter.home_mixer.product.subscribed.param.SubscribedParam.ServerMaxResultsParam import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.component_library.premarshaller.cursor.timelines.ChronologicalCursorUnmarshaller import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy import com.twitter.product_mixer.core.functional_component.common.alert.Alert import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert import com.twitter.product_mixer.core.functional_component.common.alert.P99 import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert import com.twitter.product_mixer.core.functional_component.common.alert.ThroughputAlert import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier import com.twitter.product_mixer.core.model.marshalling.request.Product import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor import com.twitter.product_mixer.core.pipeline.PipelineConfig import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig import com.twitter.product_mixer.core.product.ProductParamConfig import com.twitter.product_mixer.core.util.SortIndexBuilder import com.twitter.timelines.configapi.Params import com.twitter.timelines.render.{thriftscala => urt} import com.twitter.timelines.util.RequestCursorSerializer import com.twitter.util.Time import com.twitter.util.Try import javax.inject.Inject import javax.inject.Singleton @Singleton class SubscribedProductPipelineConfig @Inject() ( subscribedMixerPipelineConfig: SubscribedMixerPipelineConfig, subscribedParamConfig: param.SubscribedParamConfig) extends ProductPipelineConfig[HomeMixerRequest, SubscribedQuery, urt.TimelineResponse] { override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("Subscribed") override val product: Product = SubscribedProduct override val paramConfig: ProductParamConfig = subscribedParamConfig override def pipelineQueryTransformer( request: HomeMixerRequest, params: Params ): SubscribedQuery = { val context = request.productContext match { case Some(context: SubscribedProductContext) => context case _ => throw PipelineFailure(BadRequest, "SubscribedProductContext not found") } val debugOptions = request.debugParams.flatMap(_.debugOptions) /** * Unlike other clients, newly created tweets on Android have the sort index set to the current * time instead of the top sort index + 1, so these tweets get stuck at the top of the timeline * if subsequent timeline responses use the sort index from the previous response instead of * the current time. */ val pipelineCursor = request.serializedRequestCursor.flatMap { cursor => Try(UrtCursorSerializer.deserializeOrderedCursor(cursor)) .getOrElse(ChronologicalCursorUnmarshaller(RequestCursorSerializer.deserialize(cursor))) .map { case UrtOrderedCursor(_, id, Some(GapCursor), gapBoundaryId) if id.isEmpty || gapBoundaryId.isEmpty => throw PipelineFailure(MalformedCursor, "Gap Cursor bounds not defined") case topCursor @ UrtOrderedCursor(_, _, Some(TopCursor), _) => val queryTime = debugOptions.flatMap(_.requestTimeOverride).getOrElse(Time.now) topCursor.copy(initialSortIndex = SortIndexBuilder.timeToId(queryTime)) case cursor => cursor } } SubscribedQuery( params = params, clientContext = request.clientContext, features = None, pipelineCursor = pipelineCursor, requestedMaxResults = Some(params(ServerMaxResultsParam)), debugOptions = debugOptions, deviceContext = context.deviceContext, seenTweetIds = context.seenTweetIds ) } override val pipelines: Seq[PipelineConfig] = Seq(subscribedMixerPipelineConfig) override def pipelineSelector( query: SubscribedQuery ): ComponentIdentifier = subscribedMixerPipelineConfig.identifier override val alerts: Seq[Alert] = Seq( SuccessRateAlert( notificationGroup = DefaultNotificationGroup, warnPredicate = TriggerIfBelow(99.9, 20, 30), criticalPredicate = TriggerIfBelow(99.9, 30, 30), ), LatencyAlert( notificationGroup = DefaultNotificationGroup, percentile = P99, warnPredicate = TriggerIfLatencyAbove(1100.millis, 15, 30), criticalPredicate = TriggerIfLatencyAbove(1200.millis, 15, 30) ), ThroughputAlert( notificationGroup = DefaultNotificationGroup, warnPredicate = TriggerIfAbove(18000), criticalPredicate = TriggerIfAbove(20000) ) ) override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", ], exports = [ "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model/SubscribedQuery.scala ================================================ package com.twitter.home_mixer.product.subscribed.model import com.twitter.home_mixer.model.request.DeviceContext import com.twitter.home_mixer.model.request.HasDeviceContext import com.twitter.home_mixer.model.request.HasSeenTweetIds import com.twitter.home_mixer.model.request.SubscribedProduct import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.marshalling.request._ import com.twitter.product_mixer.core.pipeline.HasPipelineCursor import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.timelines.configapi.Params case class SubscribedQuery( override val params: Params, override val clientContext: ClientContext, override val pipelineCursor: Option[UrtOrderedCursor], override val requestedMaxResults: Option[Int], override val debugOptions: Option[DebugOptions], override val features: Option[FeatureMap], override val deviceContext: Option[DeviceContext], override val seenTweetIds: Option[Seq[Long]]) extends PipelineQuery with HasPipelineCursor[UrtOrderedCursor] with HasDeviceContext with HasSeenTweetIds { override val product: Product = SubscribedProduct override def withFeatureMap(features: FeatureMap): SubscribedQuery = copy(features = Some(features)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParam.scala ================================================ package com.twitter.home_mixer.product.subscribed.param import com.twitter.timelines.configapi.FSBoundedParam import com.twitter.timelines.configapi.FSParam object SubscribedParam { val SupportedClientFSName = "subscribed_supported_client" object ServerMaxResultsParam extends FSBoundedParam[Int]( name = "subscribed_server_max_results", default = 100, min = 1, max = 500 ) object EnablePostContextFeatureHydratorParam extends FSParam[Boolean]( name = "subscribed_enable_post_context_feature_hydrator", default = false ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/param/SubscribedParamConfig.scala ================================================ package com.twitter.home_mixer.product.subscribed.param import com.twitter.home_mixer.param.decider.DeciderKey import com.twitter.home_mixer.product.subscribed.param.SubscribedParam._ import com.twitter.product_mixer.core.product.ProductParamConfig import com.twitter.servo.decider.DeciderKeyName import javax.inject.Inject import javax.inject.Singleton @Singleton class SubscribedParamConfig @Inject() () extends ProductParamConfig { override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableSubscribedProduct override val supportedClientFSName: String = SupportedClientFSName override val boundedIntFSOverrides = Seq( ServerMaxResultsParam ) override val booleanFSOverrides = Seq( EnablePostContextFeatureHydratorParam ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/service/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAccessPolicy.scala ================================================ package com.twitter.home_mixer.service import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy import com.twitter.product_mixer.core.functional_component.common.access_policy.AllowedLdapGroups object HomeMixerAccessPolicy { /** * Access policies can be configured on a product-by-product basis but you may also want products * to have a common policy. */ val DefaultHomeMixerAccessPolicy: Set[AccessPolicy] = Set(AllowedLdapGroups(Set.empty[String])) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/service/HomeMixerAlertConfig.scala ================================================ package com.twitter.home_mixer.service import com.twitter.conversions.DurationOps._ import com.twitter.product_mixer.core.functional_component.common.alert.Destination import com.twitter.product_mixer.core.functional_component.common.alert.EmptyResponseRateAlert import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup import com.twitter.product_mixer.core.functional_component.common.alert.P99 import com.twitter.product_mixer.core.functional_component.common.alert.Percentile import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove import com.twitter.util.Duration /** * Notifications (email, pagerduty, etc) can be specific per-alert but it is common for multiple * products to share notification configuration. */ object HomeMixerAlertConfig { val DefaultNotificationGroup: NotificationGroup = NotificationGroup( warn = Destination(emails = Seq("")), critical = Destination(emails = Seq("")) ) object BusinessHours { val DefaultNotificationGroup: NotificationGroup = NotificationGroup( warn = Destination(emails = Seq("")), critical = Destination(emails = Seq("")) ) def defaultEmptyResponseRateAlert(warnThreshold: Double = 50, criticalThreshold: Double = 80) = EmptyResponseRateAlert( notificationGroup = DefaultNotificationGroup, warnPredicate = TriggerIfAbove(warnThreshold), criticalPredicate = TriggerIfAbove(criticalThreshold) ) def defaultSuccessRateAlert( threshold: Double = 99.5, warnDatapointsPastThreshold: Int = 20, criticalDatapointsPastThreshold: Int = 30, duration: Int = 30 ) = SuccessRateAlert( notificationGroup = DefaultNotificationGroup, warnPredicate = TriggerIfBelow(threshold, warnDatapointsPastThreshold, duration), criticalPredicate = TriggerIfBelow(threshold, criticalDatapointsPastThreshold, duration), ) def defaultLatencyAlert( latencyThreshold: Duration = 200.millis, warningDatapointsPastThreshold: Int = 15, criticalDatapointsPastThreshold: Int = 30, duration: Int = 30, percentile: Percentile = P99 ): LatencyAlert = LatencyAlert( notificationGroup = DefaultNotificationGroup, percentile = percentile, warnPredicate = TriggerIfLatencyAbove(latencyThreshold, warningDatapointsPastThreshold, duration), criticalPredicate = TriggerIfLatencyAbove(latencyThreshold, criticalDatapointsPastThreshold, duration) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/service/ScoredTweetsService.scala ================================================ package com.twitter.home_mixer.service import com.twitter.home_mixer.{thriftscala => t} import com.twitter.product_mixer.core.model.marshalling.request.Request import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry import com.twitter.stitch.Stitch import com.twitter.timelines.configapi.Params import javax.inject.Inject import javax.inject.Singleton import scala.reflect.runtime.universe._ @Singleton class ScoredTweetsService @Inject() (productPipelineRegistry: ProductPipelineRegistry) { def getScoredTweetsResponse[RequestType <: Request]( request: RequestType, params: Params )( implicit requestTypeTag: TypeTag[RequestType] ): Stitch[t.ScoredTweetsResponse] = productPipelineRegistry .getProductPipeline[RequestType, t.ScoredTweetsResponse](request.product) .process(ProductPipelineRequest(request, params)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/store/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/twitter/bijection:scrooge", "3rdparty/jvm/com/twitter/storehaus:core", "finagle/finagle-memcached/src/main/scala", "finatra/inject/inject-core/src/main/scala", "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/offheap", "media-understanding/video-summary/thrift/src/main/thrift:thrift-scala", "src/java/com/twitter/ml", "src/scala/com/twitter/ml/api:api-base", "src/scala/com/twitter/ml/api/util:datarecord", "src/scala/com/twitter/simclusters_v2/summingbird/stores", "src/scala/com/twitter/storehaus_internal/manhattan", "src/scala/com/twitter/storehaus_internal/memcache", "src/scala/com/twitter/storehaus_internal/memcache/config", "src/scala/com/twitter/storehaus_internal/offline", "src/scala/com/twitter/storehaus_internal/util", "src/thrift/com/twitter/ml/api:data-java", "src/thrift/com/twitter/timelines/realtime_aggregates:thrift-scala", "src/thrift/com/twitter/twistly:twistly-scala", "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", "stitch/stitch-core", "util-internal/util-cache/src/main/java", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/store/MediaClusterId88Store.scala ================================================ package com.twitter.home_mixer.store import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import javax.inject.Inject import javax.inject.Singleton @Singleton class MediaClusterId88Store @Inject() ( val statsReceiver: StatsReceiver, val serviceIdentifier: ServiceIdentifier) extends MediaClusterIdStoreTrait { override val appId: String = "media_understanding_embeddings_prod" override val dataset: String = "media_embedding_twitter_clip_v0_cluster_id_88" override val memcacheDest: String = "/s/cache/clip_cluster_id_88" override val keyPrefix: String = "" // Empty key prefix as per config override val storeName: String = "MediaClusterId88Store" } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/store/MediaClusterId95Store.scala ================================================ package com.twitter.home_mixer.store import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import javax.inject.Inject import javax.inject.Singleton @Singleton class MediaClusterId95Store @Inject() ( override val statsReceiver: StatsReceiver, override val serviceIdentifier: ServiceIdentifier) extends MediaClusterIdStoreTrait { override def appId: String = "media_understanding_embeddings_prod" override def dataset: String = "media_embedding_twitter_clip_v0_cluster_id_95" override def memcacheDest: String = "/s/cache/clip_cluster_id_95" override def keyPrefix: String = "" // Empty key prefix as per config override def storeName: String = "MediaClusterId95Store" } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/store/MediaClusterIdStoreTrait.scala ================================================ package com.twitter.home_mixer.store import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.storage.client.manhattan.bijections.Bijections import com.twitter.storage.client.manhattan.kv.ManhattanKVClient import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder import com.twitter.storage.client.manhattan.kv.impl.ReadOnlyKeyDescriptor import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor import com.twitter.storehaus.ReadableStore import com.twitter.storehaus_internal.memcache.MemcacheStore import com.twitter.storehaus_internal.util.ClientName import com.twitter.storehaus_internal.util.ZkEndPoint import com.twitter.bijection.Bijection trait MediaClusterIdStoreTrait { val statsReceiver: StatsReceiver val serviceIdentifier: ServiceIdentifier // Abstract methods that subclasses must implement def appId: String def dataset: String def memcacheDest: String def keyPrefix: String def storeName: String // For logging private val ManhattanDest = "/s/manhattan/nash.native-thrift" lazy val clusterIdStore: ReadableStore[Long, Long] = { val manhattanStore = createManhattanStore() val cachedStore = createCachedStore(manhattanStore) cachedStore } private def createCachedStore( underlyingStore: ReadableStore[Long, Long] ): ReadableStore[Long, Long] = { val memcacheStats = statsReceiver.scope(s"memcache_${keyPrefix}") val underlyingCacheClient = MemcacheStore.memcachedClient( name = ClientName(serviceIdentifier.name), dest = ZkEndPoint(memcacheDest), statsReceiver = memcacheStats, serviceIdentifier = serviceIdentifier, timeout = 80.milliseconds ) val memcacheStore = ObservedMemcachedReadableStore.fromCacheClient( backingStore = underlyingStore, cacheClient = underlyingCacheClient, ttl = 3.days, // From config: foundTtl = 3.days )( valueInjection = Bijections.long2ByteArray, statsReceiver = memcacheStats, keyToString = { key: Long => val cacheKey = if (keyPrefix.isEmpty) key.toString else s"${keyPrefix}_${key}" cacheKey } ) memcacheStore } private def createManhattanStore(): ReadableStore[Long, Long] = { // Use proper key and value descriptors with correct encodings // Sign flip for NativeEncoding (flip high bit to match server-side ordering) def signFlip(a: Array[Byte]): Array[Byte] = { a(0) = (a(0) ^ 0x80.toByte).toByte; a } val signBijection = Bijection.build[Array[Byte], Array[Byte]](signFlip _)(signFlip _) val keyInjection = Bijections.long2ByteArray.andThen(signBijection).andThen(Bijections.byteArray2Buf) val keyDesc = ReadOnlyKeyDescriptor(keyInjection) val datasetKey = keyDesc.withDataset(dataset) val valueInjection = Bijections.long2ByteArray.andThen(Bijections.byteArray2Buf) val valueDesc = ValueDescriptor(valueInjection) val client = ManhattanKVClient( appId = appId, dest = ManhattanDest, mtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier), label = storeName ) val endpoint = ManhattanKVEndpointBuilder(client) .maxRetryCount(3) .defaultMaxTimeout(120.milliseconds) .build() new ReadableStore[Long, Long] { override def get(key: Long) = { import com.twitter.stitch.Stitch val future = Stitch.run(endpoint.get(datasetKey.withPkey(key), valueDesc)) future.map(_.map(_.contents)) } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/store/RTAMHStore.scala ================================================ package com.twitter.home_mixer.store import com.twitter.bijection.Injection import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.home_mixer.store.RTAManhattanRealGraphKVDescriptor._ import com.twitter.io.Buf import com.twitter.ml.api.DataRecord import com.twitter.stitch.Stitch import com.twitter.storage.client.manhattan.bijections.Bijections import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint import com.twitter.storage.client.manhattan.kv.impl.ReadOnlyKeyDescriptor import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import com.twitter.ml.api.{thriftscala => mlThrift} import com.twitter.timelines.realtime_aggregates.{thriftscala => thrift} import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions._ object RTAManhattanRealGraphKVDescriptor { val datasetName = "timelines_real_time_aggregates_0" val keyInjection: Injection[thrift.AggregationKey, Buf] = BinaryScalaCodec(thrift.AggregationKey).andThen(Bijections.byteArray2Buf) val keyDesc = ReadOnlyKeyDescriptor(keyInjection) val datasetKey = keyDesc.withDataset(datasetName) val valueInjection = BinaryScalaCodec(mlThrift.DataRecord).andThen(Bijections.byteArray2Buf) val valueDesc = ValueDescriptor(valueInjection) } /** * */ class RTAMHStore(manhattanKVEndpoint: ManhattanKVEndpoint) extends ReadableStore[thrift.AggregationKey, DataRecord] { override def get(key: thrift.AggregationKey): Future[Option[DataRecord]] = Stitch .run(manhattanKVEndpoint.get(datasetKey.withPkey(key), valueDesc)) .map(_.map(mhResponse => mhResponse.contents).map(scalaDataRecord2JavaDataRecord)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/store/RealGraphInNetworkScoresStore.scala ================================================ package com.twitter.home_mixer.store import com.twitter.bijection.Injection import com.twitter.home_mixer.store.ManhattanRealGraphKVDescriptor._ import com.twitter.stitch.Stitch import com.twitter.storage.client.manhattan.bijections.Bijections import com.twitter.storage.client.manhattan.bijections.Bijections.BinaryScalaInjection import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint import com.twitter.storage.client.manhattan.kv.impl.ReadOnlyKeyDescriptor import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor import com.twitter.storehaus.ReadableStore import com.twitter.util.Future import com.twitter.wtf.candidate.{thriftscala => wtf} object ManhattanRealGraphKVDescriptor { implicit val byteArray2Buf = Bijections.BytesBijection val realGraphDatasetName = "real_graph_scores_in_v1" val keyInjection = Injection.connect[Long, Array[Byte]].andThen(Bijections.BytesInjection) val keyDesc = ReadOnlyKeyDescriptor(keyInjection) val valueDesc = ValueDescriptor(BinaryScalaInjection(wtf.CandidateSeq)) val realGraphDatasetKey = keyDesc.withDataset(realGraphDatasetName) } /** * Hydrates real graph in network scores for a viewer */ class RealGraphInNetworkScoresStore(manhattanKVEndpoint: ManhattanKVEndpoint) extends ReadableStore[Long, Seq[wtf.Candidate]] { override def get(viewerId: Long): Future[Option[Seq[wtf.Candidate]]] = Stitch .run(manhattanKVEndpoint.get(realGraphDatasetKey.withPkey(viewerId), valueDesc)) .map(_.map(mhResponse => mhResponse.contents.candidates)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/store/TweetWatchTimeMetadataStore.scala ================================================ package com.twitter.home_mixer.store import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.storage.client.manhattan.bijections.Bijections import com.twitter.storage.client.manhattan.kv.ManhattanKVClient import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder import com.twitter.storage.client.manhattan.kv.impl.Component import com.twitter.storage.client.manhattan.kv.impl.KeyDescriptor import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor import com.twitter.storehaus.ReadableStore import com.twitter.storehaus_internal.memcache.MemcacheStore import com.twitter.storehaus_internal.util.ClientName import com.twitter.storehaus_internal.util.ZkEndPoint import com.twitter.twistly.thriftscala.VideoViewEngagementType import com.twitter.twistly.thriftscala.WatchTimeMetadata import com.twitter.bijection.Injection import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.io.Buf import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton import scala.util.Try @Singleton class TweetWatchTimeMetadataStore @Inject() ( statsReceiver: StatsReceiver, serviceIdentifier: ServiceIdentifier) { private val ManhattanDest = "/s/manhattan/nash.native-thrift" private val AppId = "uss_prod" private val Dataset = "video_watch_time_metadata" private val MemcacheDest = "/s/cache/tweet_aggregated_watch_time" private val KeyPrefix = "" // Empty key prefix as per config lazy val tweetWatchTimeMetadataStore: ReadableStore[ (Long, VideoViewEngagementType), WatchTimeMetadata ] = { val manhattanStore = createManhattanStore() val cachedStore = createCachedStore(manhattanStore) cachedStore } private def createCachedStore( underlyingStore: ReadableStore[(Long, VideoViewEngagementType), WatchTimeMetadata] ): ReadableStore[(Long, VideoViewEngagementType), WatchTimeMetadata] = { val memcacheStats = statsReceiver.scope(s"memcache_${KeyPrefix}") val underlyingCacheClient = MemcacheStore.memcachedClient( name = ClientName(serviceIdentifier.name), dest = ZkEndPoint(MemcacheDest), statsReceiver = memcacheStats, serviceIdentifier = serviceIdentifier, timeout = 80.milliseconds ) val memcacheStore = ObservedMemcachedReadableStore.fromCacheClient( backingStore = underlyingStore, cacheClient = underlyingCacheClient, ttl = 10.minutes, )( valueInjection = BinaryScalaCodec(WatchTimeMetadata), statsReceiver = memcacheStats, keyToString = { key: (Long, VideoViewEngagementType) => val cacheKey = if (KeyPrefix.isEmpty) s"${key._1}_${key._2}" else s"${KeyPrefix}_${key._1}_${key._2}" cacheKey } ) memcacheStore } private def createManhattanStore( ): ReadableStore[(Long, VideoViewEngagementType), WatchTimeMetadata] = { val pkeyInjection: Injection[Long, Buf] = Bijections.long2ByteArray.andThen(Bijections.BytesBijection) val lkeyInjection: Injection[VideoViewEngagementType, Buf] = Injection .build[VideoViewEngagementType, Int](_.value)(i => Try(VideoViewEngagementType(i))) .andThen(Bijections.int2ByteArray) .andThen(Bijections.BytesBijection) val keyDesc = KeyDescriptor(Component(pkeyInjection), Component(lkeyInjection)).withDataset(Dataset) val valueInjection: Injection[WatchTimeMetadata, Buf] = Bijections.BinaryScalaInjection(WatchTimeMetadata) val valueDesc = ValueDescriptor(valueInjection) val client = ManhattanKVClient( appId = AppId, dest = ManhattanDest, mtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier), label = "TweetWatchTimeMetadata" ) val endpoint = ManhattanKVEndpointBuilder(client) .maxRetryCount(3) .defaultMaxTimeout(120.milliseconds) .build() new ReadableStore[(Long, VideoViewEngagementType), WatchTimeMetadata] { override def get(key: (Long, VideoViewEngagementType)) = { val (tweetId, engType) = key val mhKey = keyDesc.withPkey(tweetId).withLkey(engType) Stitch.run(endpoint.get(mhKey, valueDesc)).map { opt => opt.map { mv => mv.contents } } } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/store/TwhinEmbeddingsStore.scala ================================================ package com.twitter.home_mixer.store import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.conversions.DurationOps._ import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.hermit.store.offheap.Codecs import com.twitter.hermit.store.offheap.OffheapCachedReadableStore import com.twitter.scrooge.ThriftStruct import com.twitter.simclusters_v2.summingbird.stores.ManhattanFromStratoStore import com.twitter.simclusters_v2.thriftscala.PersistentTwhinTweetEmbedding import com.twitter.simclusters_v2.thriftscala.PersistentTwhinUserEmbedding import com.twitter.simclusters_v2.{thriftscala => sim} import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams import com.twitter.storehaus.ReadableStore import com.twitter.storehaus_internal.memcache.MemcacheStore import com.twitter.storehaus_internal.util._ import javax.inject.Inject import javax.inject.Singleton import com.twitter.offheap.KeyHashFunction import com.twitter.offheap.MemoryWriter import com.twitter.offheap.ObjectCodec import com.twitter.offheap.OffheapReader import com.twitter.simclusters_v2.thriftscala.TwhinTweetEmbedding @Singleton() class TwhinEmbeddingsStore @Inject() ( statsReceiver: StatsReceiver, serviceIdentifier: ServiceIdentifier) { val ManhattanNashDest = "/s/manhattan/nash.native-thrift" val TwhinEmbeddingsProdAppId = "twhin_embeddings_prod" val UserPositiveDataset = "twhin_user_positive_embeddings" val RebuildUserPositiveDataset = "twhin_rebuild_user_rt_pos_emb" val UserNegativeDataset = "twhin_user_negative_embeddings" val TweetDataset = "twhin_tweet_embeddings" val TweetRebuildDataset = "twhin_rebuild_tweet_rt_emb" val VideoDataset = "twhin_video_embeddings" val MemcacheTweetDest = "/s/cache/twhin_embeddings" val MemcacheVideoDest = "/s/cache/twhin_video_embeddings" val KeyPrefixTweet = "twhin_tweets" val KeyPrefixTweetRebuild = "twhin_tweets_rebuild" val KeyPrefixVideo = "twhin_videos" val InMemoryCachePrefix = "in_memory_cache" val MinEngagementCount = 16 val IsProdEnv = serviceIdentifier.environment == "prod" /** * We do not generate the tweet or video embedding if the number of recent engagements * is < `MinEngagementCount`. This is based on prior Simcluster embedding aggregation * experience and in order to be consistent with the Strato column * strato/config/columns/recommendations/twhin/CachedTwhinTweetEmbeddings.Tweet.strato */ private def normalizeByCount( persistentEmbedding: sim.PersistentTwhinTweetEmbedding ): sim.TwhinTweetEmbedding = { val embedding = persistentEmbedding.embedding.embedding val updatedEmbedding = if (persistentEmbedding.updatedCount < MinEngagementCount) embedding.map(_ => 0.0) else embedding.map(_ / persistentEmbedding.updatedCount) sim.TwhinTweetEmbedding(updatedEmbedding) } private def createManhattanStore[T <: ThriftStruct: Manifest]( dataset: String ): ReadableStore[Long, T] = { ManhattanFromStratoStore .createPersistentTwhinStore[T]( dataset = dataset, mhMtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier), statsReceiver = statsReceiver, appId = TwhinEmbeddingsProdAppId, dest = ManhattanNashDest ).composeKeyMapping((_, 0L)) } private def createManhattanVersionedStore[T <: ThriftStruct: Manifest]( dataset: String ): ReadableStore[(Long, Long), T] = { ManhattanFromStratoStore .createPersistentTwhinStore[T]( dataset = dataset, mhMtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier), statsReceiver = statsReceiver, appId = TwhinEmbeddingsProdAppId, dest = ManhattanNashDest ).composeKeyMapping[(Long, Long)](key => key) } private def createCachedStore[K]( underlyingStore: ReadableStore[K, sim.TwhinTweetEmbedding], cacheDest: String, keyPrefix: String, keyCodec: ObjectCodec[K], keyHashFunction: KeyHashFunction[K] ): ReadableStore[K, sim.TwhinTweetEmbedding] = { val scopedStatsReceiver = statsReceiver.scope(keyPrefix) val underlyingCacheClient = MemcacheStore.memcachedClient( name = ClientName(keyPrefix), dest = ZkEndPoint(cacheDest), statsReceiver = scopedStatsReceiver, serviceIdentifier = serviceIdentifier, timeout = 80.milliseconds ) val memcacheStore = ObservedMemcachedReadableStore.fromCacheClient( backingStore = underlyingStore, cacheClient = underlyingCacheClient, ttl = 15.minutes, asyncUpdate = IsProdEnv )( valueInjection = BinaryScalaCodec(sim.TwhinTweetEmbedding), statsReceiver = scopedStatsReceiver, keyToString = { key: K => s"${keyPrefix}_${key}" } ) OffheapCachedReadableStore.fromCache( memcacheStore, tableSize = 256 * 1024, capacity = 256 * 1024 * 2048, keyHashFunction = keyHashFunction, keyCodec = keyCodec, valueCodec = TwhinEmbeddingsStore.TwhinTweetEmbeddingCodec, ttl = 1.minutes, statsReceiver = scopedStatsReceiver ) } val mhUserPositiveStore: ReadableStore[Long, sim.TwhinTweetEmbedding] = createManhattanStore[PersistentTwhinUserEmbedding](UserPositiveDataset).mapValues(_.embedding) val mhRebuildUserPositiveStore: ReadableStore[(Long, Long), sim.TwhinTweetEmbedding] = createManhattanVersionedStore[PersistentTwhinUserEmbedding](RebuildUserPositiveDataset) .mapValues(_.embedding) val mhUserNegativeStore: ReadableStore[Long, sim.TwhinTweetEmbedding] = createManhattanStore[PersistentTwhinUserEmbedding](UserNegativeDataset).mapValues(_.embedding) val mhTweetStore: ReadableStore[Long, sim.TwhinTweetEmbedding] = createManhattanStore[PersistentTwhinTweetEmbedding](TweetDataset).mapValues(normalizeByCount) val mhVideoStore: ReadableStore[Long, sim.TwhinTweetEmbedding] = createManhattanStore[PersistentTwhinTweetEmbedding](VideoDataset).mapValues(normalizeByCount) val cachedTweetStore: ReadableStore[Long, TwhinTweetEmbedding] = createCachedStore( mhTweetStore, MemcacheTweetDest, KeyPrefixTweet, Codecs.LongKeyCodec, key => java.lang.Long.hashCode(key) ) val mhTweetRebuildStore: ReadableStore[(Long, Long), sim.TwhinTweetEmbedding] = createManhattanVersionedStore[PersistentTwhinTweetEmbedding](TweetRebuildDataset) .mapValues(normalizeByCount) val cachedTweetRebuildStore: ReadableStore[(Long, Long), TwhinTweetEmbedding] = createCachedStore[(Long, Long)]( mhTweetRebuildStore, MemcacheTweetDest, KeyPrefixTweetRebuild, Codecs.LongTupleKeyCodec, key => key.hashCode() ) val cachedVideoStore: ReadableStore[Long, TwhinTweetEmbedding] = createCachedStore( mhVideoStore, MemcacheVideoDest, KeyPrefixVideo, Codecs.LongKeyCodec, key => java.lang.Long.hashCode(key) ) } object TwhinEmbeddingsStore { object TwhinTweetEmbeddingCodec extends ObjectCodec[sim.TwhinTweetEmbedding] { override def size(decoded: sim.TwhinTweetEmbedding): Int = { Codecs.SeqDoubleCodec.size(decoded.embedding) } override def encode(decoded: sim.TwhinTweetEmbedding, writer: MemoryWriter): Unit = { Codecs.SeqDoubleCodec.encode(decoded.embedding, writer) } override def decode(reader: OffheapReader): sim.TwhinTweetEmbedding = { sim.TwhinTweetEmbedding( embedding = Codecs.SeqDoubleCodec.decode(reader) ) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/store/VideoEmbeddingMHStore.scala ================================================ package com.twitter.home_mixer.store import com.twitter.conversions.DurationOps.richDurationFromInt import com.twitter.finagle.mtls.authentication.ServiceIdentifier import com.twitter.finagle.stats.StatsReceiver import com.twitter.hermit.store.common.ObservedMemcachedReadableStore import com.twitter.storage.client.manhattan.bijections.Bijections import com.twitter.storage.client.manhattan.kv.ManhattanKVClient import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder import com.twitter.storage.client.manhattan.kv.impl.ReadOnlyKeyDescriptor import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor import com.twitter.storehaus.ReadableStore import com.twitter.storehaus_internal.memcache.MemcacheStore import com.twitter.storehaus_internal.util.ClientName import com.twitter.storehaus_internal.util.ZkEndPoint import com.twitter.media_understanding.video_summary.thriftscala.VideoEmbedding import com.twitter.bijection.Injection import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.io.Buf import com.twitter.stitch.Stitch import javax.inject.Inject import javax.inject.Singleton @Singleton class VideoEmbeddingMHStore @Inject() ( statsReceiver: StatsReceiver, serviceIdentifier: ServiceIdentifier) { private val ManhattanDest = "/s/manhattan/nash.native-thrift" private val AppId = "media_understanding_video_summary" private val Dataset = "video_summary_embedding" private val MemcacheDest = "/s/cache/video_summary_embedding" private val KeyPrefix = "summary_embedding" lazy val videoEmbeddingMHStore: ReadableStore[Long, VideoEmbedding] = { val manhattanStore = createManhattanStore() val cachedStore = createCachedStore(manhattanStore) cachedStore } private def createCachedStore( underlyingStore: ReadableStore[Long, VideoEmbedding] ): ReadableStore[Long, VideoEmbedding] = { val memcacheStats = statsReceiver.scope(s"memcache_${KeyPrefix}") val underlyingCacheClient = MemcacheStore.memcachedClient( name = ClientName(serviceIdentifier.name), dest = ZkEndPoint(MemcacheDest), statsReceiver = memcacheStats, serviceIdentifier = serviceIdentifier, timeout = 80.milliseconds ) val memcacheStore = ObservedMemcachedReadableStore.fromCacheClient( backingStore = underlyingStore, cacheClient = underlyingCacheClient, ttl = 7.days, // foundTtl from config )( valueInjection = BinaryScalaCodec(VideoEmbedding), statsReceiver = memcacheStats, keyToString = { key: Long => s"${KeyPrefix}_${key}" } ) memcacheStore } private def createManhattanStore(): ReadableStore[Long, VideoEmbedding] = { val keyInjection: Injection[Long, Buf] = Bijections.long2ByteArray.andThen(Bijections.BytesBijection) val keyDesc = ReadOnlyKeyDescriptor(keyInjection) val datasetKey = keyDesc.withDataset(Dataset) val valueInjection: Injection[VideoEmbedding, Buf] = Bijections.BinaryScalaInjection(VideoEmbedding) val valueDesc = ValueDescriptor(valueInjection) val client = ManhattanKVClient( appId = AppId, dest = ManhattanDest, mtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier), label = "VideoEmbeddingMH" ) val endpoint = ManhattanKVEndpointBuilder(client) .maxRetryCount(3) .defaultMaxTimeout(120.milliseconds) .build() new ReadableStore[Long, VideoEmbedding] { override def get(key: Long) = { val mhKey = datasetKey.withPkey(key) Stitch.run(endpoint.get(mhKey, valueDesc)).map { opt => opt.map { mv => mv.contents } } } } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/io/grpc:grpc-protobuf", "3rdparty/jvm/io/grpc:grpc-stub", "finagle-internal/finagle-grpc/src/main/scala", "finatra/inject/inject-utils/src/main/scala", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/thrift/src/main/thrift:thrift-scala", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", "servo/repo/src/main/scala", "snowflake/src/main/scala/com/twitter/snowflake/id", "src/java/com/twitter/ml/api:api-base", "src/java/com/twitter/search/common/util/lang", "src/scala/com/twitter/ml/api/util", "src/thrift/com/twitter/search/common:constants-java", "src/thrift/com/twitter/service/metastore/gen:thrift-scala", "storage/clients/manhattan/client/src/main/scala", "strato/config/src/thrift/com/twitter/strato/columns/content_understanding:content_understanding-scala", "user-signal-service/thrift/src/main/thrift:thrift-scala", "user_history_transformer/service/src/main/java/com/x/user_action_sequence", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CachedScoredTweetsHelper.scala ================================================ package com.twitter.home_mixer.util import com.twitter.home_mixer.model.HomeFeatures.CachedScoredTweetsFeature import com.twitter.home_mixer.{thriftscala => hmt} import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweets import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier import com.twitter.snowflake.id.SnowflakeId import com.twitter.util.Time object CachedScoredTweetsHelper { def tweetImpressionsAndCachedScoredTweets( features: FeatureMap, candidatePipelineIdentifier: CandidatePipelineIdentifier ): Seq[Long] = { val tweetImpressions = features.getOrElse(ImpressedTweets, Seq.empty).toSet val cachedScoredTweets = features .getOrElse(CachedScoredTweetsFeature, Seq.empty) .filter { tweet => tweet.candidatePipelineIdentifier.exists( CandidatePipelineIdentifier(_).equals(candidatePipelineIdentifier)) }.map(_.tweetId) (tweetImpressions ++ cachedScoredTweets).toSeq } def tweetImpressionsAndCachedScoredTweetsInRange( features: FeatureMap, candidatePipelineIdentifier: CandidatePipelineIdentifier, maxNumImpressions: Int, sinceTime: Time, untilTime: Time ): Seq[Long] = tweetImpressionsAndCachedScoredTweets(features, candidatePipelineIdentifier) .filter { tweetId => SnowflakeId.isSnowflakeId(tweetId) } .filter { tweetId => val creationTime = SnowflakeId.timeFromId(tweetId) sinceTime <= creationTime && untilTime >= creationTime }.take(maxNumImpressions) def unseenCachedScoredTweets( features: FeatureMap ): Seq[hmt.ScoredTweet] = { val seenTweetIds = features.getOrElse(ImpressedTweets, Seq.empty).toSet features .getOrElse(CachedScoredTweetsFeature, Seq.empty) .filter(tweet => !seenTweetIds.contains(tweet.tweetId)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/CandidatesUtil.scala ================================================ package com.twitter.home_mixer.util import com.twitter.escherbird.common.thriftscala.QualifiedId import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.GrokVideoMetadataFeature import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.MediaUnderstandingAnnotationIdsFeature import com.twitter.home_mixer.model.HomeFeatures.RepliedByEngagerIdsFeature import com.twitter.home_mixer.model.HomeFeatures.RetweetedByEngagerIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.VideoAspectRatioFeature import com.twitter.product_mixer.component_library.model.candidate.CursorCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.UniversalNoun import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure import com.twitter.product_mixer.core.pipeline.pipeline_failure.UnexpectedCandidateResult import scala.reflect.ClassTag object CandidatesUtil { def getItemCandidates(candidates: Seq[CandidateWithDetails]): Seq[ItemCandidateWithDetails] = { candidates.collect { case item: ItemCandidateWithDetails if !item.isCandidateType[CursorCandidate] => Seq(item) case module: ModuleCandidateWithDetails => module.candidates }.flatten } def getItemCandidatesWithOnlyModuleLast( candidates: Seq[CandidateWithDetails] ): Seq[ItemCandidateWithDetails] = { candidates.collect { case item: ItemCandidateWithDetails if !item.isCandidateType[CursorCandidate] => item case module: ModuleCandidateWithDetails => module.candidates.last } } def containsType[CandidateType <: UniversalNoun[_]]( candidates: Seq[CandidateWithDetails] )( implicit tag: ClassTag[CandidateType] ): Boolean = candidates.exists { case ItemCandidateWithDetails(_: CandidateType, _, _) => true case module: ModuleCandidateWithDetails => module.candidates.head.isCandidateType[CandidateType]() case _ => false } def getOriginalTweetId(candidate: CandidateWithFeatures[TweetCandidate]): Long = { if (candidate.features.getOrElse(IsRetweetFeature, false)) candidate.features.getOrElse(SourceTweetIdFeature, None).getOrElse(candidate.candidate.id) else candidate.candidate.id } def getOriginalTweetId(candidate: TweetCandidate, features: FeatureMap): Long = { if (features.getOrElse(IsRetweetFeature, false)) features.getOrElse(SourceTweetIdFeature, None).getOrElse(candidate.id) else candidate.id } def getOriginalTweetId(candidate: CandidateWithDetails): Long = { candidate match { case ItemCandidateWithDetails(candidate: TweetCandidate, _, feautres) => getOriginalTweetId(candidate, feautres) case _ => throw PipelineFailure(UnexpectedCandidateResult, "Invalid candidate type") } } def getOriginalTweetId(candidateId: Long, features: FeatureMap): Long = { if (features.getOrElse(IsRetweetFeature, false)) features.getOrElse(SourceTweetIdFeature, None).getOrElse(candidateId) else candidateId } def getOriginalAuthorId(candidateFeatures: FeatureMap): Option[Long] = if (candidateFeatures.getOrElse(IsRetweetFeature, false)) candidateFeatures.getOrElse(SourceUserIdFeature, None) else candidateFeatures.getOrElse(AuthorIdFeature, None) def isOriginalTweet(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = !candidate.features.getOrElse(IsRetweetFeature, false) && candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty def isOriginalTweet(candidate: CandidateWithDetails): Boolean = !candidate.features.getOrElse(IsRetweetFeature, false) && candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty def getEngagerUserIds( candidateFeatures: FeatureMap ): Seq[Long] = { candidateFeatures.getOrElse(FavoritedByUserIdsFeature, Seq.empty) ++ candidateFeatures.getOrElse(RetweetedByEngagerIdsFeature, Seq.empty) ++ candidateFeatures.getOrElse(RepliedByEngagerIdsFeature, Seq.empty) } def getMediaUnderstandingAnnotationIds( candidateFeatures: FeatureMap ): Seq[Long] = { if (candidateFeatures.get(HasImageFeature)) candidateFeatures.getOrElse(MediaUnderstandingAnnotationIdsFeature, Seq.empty) else Seq.empty } def getTweetIdAndSourceId(candidate: CandidateWithFeatures[TweetCandidate]): Seq[Long] = Seq(candidate.candidate.id) ++ candidate.features.getOrElse(SourceTweetIdFeature, None) def isAuthoredByViewer(query: PipelineQuery, candidateFeatures: FeatureMap): Boolean = candidateFeatures.getOrElse(AuthorIdFeature, None).contains(query.getRequiredUserId) || (candidateFeatures.getOrElse(IsRetweetFeature, false) && candidateFeatures.getOrElse(SourceUserIdFeature, None).contains(query.getRequiredUserId)) def getCandidateTopicAndAspectRatio( candidate: CandidateWithDetails ): (Option[QualifiedId], Boolean) = { val video = candidate.features.getOrElse(GrokVideoMetadataFeature, None) val optionalQualifiedId = video.flatMap { _.entities.map { entities => entities.maxBy(entity => entity.score.getOrElse(0.0))._1 } } val aspectRatio = candidate.features.getOrElse(VideoAspectRatioFeature, None).exists(_ > 1.0) (optionalQualifiedId, aspectRatio) } val reverseChronTweetsOrdering: Ordering[CandidateWithDetails] = Ordering.by[CandidateWithDetails, Long] { case ItemCandidateWithDetails(candidate: TweetCandidate, _, _) => -candidate.id case ModuleCandidateWithDetails(candidates, _, _) if candidates.nonEmpty => -candidates.last.candidateIdLong case _ => throw PipelineFailure(UnexpectedCandidateResult, "Invalid candidate type") } val scoreOrdering: Ordering[CandidateWithDetails] = Ordering.by[CandidateWithDetails, Double] { case ItemCandidateWithDetails(_, _, features) => -features.getOrElse(ScoreFeature, None).getOrElse(0.0) case ModuleCandidateWithDetails(candidates, _, _) => -candidates.last.features.getOrElse(ScoreFeature, None).getOrElse(0.0) case _ => throw PipelineFailure(UnexpectedCandidateResult, "Invalid candidate type") } val conversationModuleTweetsOrdering: Ordering[CandidateWithDetails] = Ordering.by[CandidateWithDetails, Long] { case ItemCandidateWithDetails(candidate: TweetCandidate, _, _) => candidate.id case _ => throw PipelineFailure(UnexpectedCandidateResult, "Only Item candidate expected") } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/DataRecordUtil.scala ================================================ package com.twitter.home_mixer.util import com.twitter.ml.api.DataRecord import com.twitter.ml.api.FeatureContext import com.twitter.ml.api.util.SRichDataRecord import com.twitter.ml.api.Feature import java.lang.{Double => JDouble} object DataRecordUtil { def applyRename( dataRecord: DataRecord, featureContext: FeatureContext, renamedFeatureContext: FeatureContext, featureRenamingMap: Map[Feature[_], Feature[_]] ): DataRecord = { val richFullDr = new SRichDataRecord(dataRecord, featureContext) val richNewDr = new SRichDataRecord(new DataRecord, renamedFeatureContext) val featureIterator = featureContext.iterator() featureIterator.forEachRemaining { feature => if (richFullDr.hasFeature(feature)) { val renamedFeature = featureRenamingMap.getOrElse(feature, feature) val typedFeature = feature.asInstanceOf[Feature[JDouble]] val typedRenamedFeature = renamedFeature.asInstanceOf[Feature[JDouble]] richNewDr.setFeatureValue(typedRenamedFeature, richFullDr.getFeatureValue(typedFeature)) } } richNewDr.getRecord } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/InjectionTransformer.scala ================================================ package com.twitter.home_mixer.util import com.twitter.bijection.Injection import com.twitter.io.Buf import com.twitter.servo.util.Transformer import com.twitter.storage.client.manhattan.bijections.Bijections import com.twitter.util.Return import com.twitter.util.Try import java.nio.ByteBuffer object InjectionTransformerImplicits { implicit class ByteArrayInjectionToByteBufferTransformer[A](baInj: Injection[A, Array[Byte]]) { private val bbInj: Injection[A, ByteBuffer] = baInj .andThen(Bijections.byteArray2Buf) .andThen(Bijections.byteBuffer2Buf.inverse) def toByteBufferTransformer(): Transformer[A, ByteBuffer] = new InjectionTransformer(bbInj) def toByteArrayTransformer(): Transformer[A, Array[Byte]] = new InjectionTransformer(baInj) } implicit class BufInjectionToByteBufferTransformer[A](bufInj: Injection[A, Buf]) { private val bbInj: Injection[A, ByteBuffer] = bufInj.andThen(Bijections.byteBuffer2Buf.inverse) private val baInj: Injection[A, Array[Byte]] = bufInj.andThen(Bijections.byteArray2Buf.inverse) def toByteBufferTransformer(): Transformer[A, ByteBuffer] = new InjectionTransformer(bbInj) def toByteArrayTransformer(): Transformer[A, Array[Byte]] = new InjectionTransformer(baInj) } implicit class ByteBufferInjectionToByteBufferTransformer[A](bbInj: Injection[A, ByteBuffer]) { private val baInj: Injection[A, Array[Byte]] = bbInj.andThen(Bijections.bb2ba) def toByteBufferTransformer(): Transformer[A, ByteBuffer] = new InjectionTransformer(bbInj) def toByteArrayTransformer(): Transformer[A, Array[Byte]] = new InjectionTransformer(baInj) } } class InjectionTransformer[A, B](inj: Injection[A, B]) extends Transformer[A, B] { override def to(a: A): Try[B] = Return(inj(a)) override def from(b: B): Try[A] = Try.fromScala(inj.invert(b)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/LanguageCode.scala ================================================ package com.twitter.home_mixer.util object LanguageCode { final val Japanese = "jp" final val English = "en" final val Unknown = "zxx" val AllowedLanguageCodes = Set( "art", // Emojis "qam", // Mentions "qct", // Cashtag "qht", // Hashtag "qme", // Multiple entities "qst", // Short text "und", // Undefined "zxx" // Links ) val languageToISO: Map[String, String] = Map( "arabic" -> "ar", "danish" -> "da", "german" -> "de", "greek" -> "el", "english" -> "en", "esperanto" -> "eo", "spanish" -> "es", "persian" -> "fa", "finnish" -> "fi", "french" -> "fr", "hebrew" -> "he", "hungarian" -> "hu", "indonesian" -> "id", "icelandic" -> "is", "italian" -> "it", "japanese" -> "ja", "korean" -> "ko", "lithuanian" -> "lt", "dutch" -> "nl", "norwegian" -> "no", "polish" -> "pl", "portuguese" -> "pt", "russian" -> "ru", "swedish" -> "sv", "thai" -> "th", "urdu" -> "ur", "chinese" -> "zh", "turkish" -> "tr", "tagalog" -> "tl", "hindi" -> "hi", "malay" -> "ms", "amharic" -> "am", "bengali" -> "bn", "tibetan" -> "bo", "dhivehi" -> "dv", "gujarati" -> "gu", "armenian" -> "hy", "inuktitut" -> "iu", "georgian" -> "ka", "khmer" -> "km", "kannada" -> "kn", "lao" -> "lo", "malayalam" -> "ml", "myanmar" -> "my", "oriya" -> "or", "panjabi" -> "pa", "sinhala" -> "si", "tamil" -> "ta", "telugu" -> "te", "vietnamese" -> "vi", "bulgarian" -> "bg", "nepali" -> "ne", "estonian" -> "et", "haitian" -> "ht", "latvian" -> "lv", "slovak" -> "sk", "slovenian" -> "sl", "ukrainian" -> "uk", "basque" -> "eu", "bosnian" -> "bs", "catalan" -> "ca", "croatian" -> "hr", "czech" -> "cs", "hindi latin" -> "hi", "marathi" -> "mr", "pashto" -> "ps", "romanian" -> "ro", "serbian" -> "sr", "sindhi" -> "sd", "welsh" -> "cy", "uyghur" -> "ug" ) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/LanguageUtil.scala ================================================ package com.twitter.home_mixer.util import com.twitter.search.common.constants.{thriftscala => scc} import com.twitter.search.common.util.lang.ThriftLanguageUtil import com.twitter.service.metastore.gen.{thriftscala => smg} object LanguageUtil { private val DefaultMinProducedLanguageRatio = 0.05 private val DefaultMinConsumedLanguageConfidence = 0.8 /** * Computes a list of languages based on UserLanguages information retrieved from Metastore. * * The list is sorted in descending order of confidence score associated with each language. * That is, language with highest confidence value is in index 0. */ def computeLanguages( userLanguages: smg.UserLanguages, minProducedLanguageRatio: Double = DefaultMinProducedLanguageRatio, minConsumedLanguageConfidence: Double = DefaultMinConsumedLanguageConfidence ): Seq[scc.ThriftLanguage] = { val languageConfidenceMap = computeLanguageConfidenceMap( userLanguages, minProducedLanguageRatio, minConsumedLanguageConfidence ) languageConfidenceMap.toSeq.sortBy(-_._2).map(_._1) // _1 = language, _2 = score } /** * Computes confidence map based on UserLanguages information retrieved from Metastore. * where, * key = language code * value = level of confidence that the language is applicable to a user. */ private def computeLanguageConfidenceMap( userLanguages: smg.UserLanguages, minProducedLanguageRatio: Double, minConsumedLanguageConfidence: Double ): Map[scc.ThriftLanguage, Double] = { val producedLanguages = getLanguageMap(userLanguages.produced) val consumedLanguages = getLanguageMap(userLanguages.consumed) val languages = (producedLanguages.keys ++ consumedLanguages.keys).toSet var maxConfidence = 0.0 val confidenceMap = languages.map { language => val produceRatio = producedLanguages .get(language) .map { score => if (score < minProducedLanguageRatio) 0.0 else score } .getOrElse(0.0) val consumeConfidence = consumedLanguages .get(language) .map { score => if (score < minConsumedLanguageConfidence) 0.0 else score } .getOrElse(0.0) val overallConfidence = (0.3 + 4 * produceRatio) * (0.1 + consumeConfidence) maxConfidence = Math.max(maxConfidence, overallConfidence) (language -> overallConfidence) }.toMap val normalizedConfidenceMap = if (maxConfidence > 0) { confidenceMap.map { case (language, confidenceScore) => val normalizedScore = (confidenceScore / maxConfidence * 0.9) + 0.1 (language -> normalizedScore) } } else { confidenceMap } normalizedConfidenceMap } private def getLanguageMap( scoredLanguages: Seq[smg.ScoredString] ): Map[scc.ThriftLanguage, Double] = { scoredLanguages.flatMap { scoredLanguage => getThriftLanguage(scoredLanguage.item).map { language => (language -> scoredLanguage.weight) } }.toMap } private def getThriftLanguage(languageName: String): Option[scc.ThriftLanguage] = { val languageOrdinal = ThriftLanguageUtil.getThriftLanguageOf(languageName).ordinal val language = scc.ThriftLanguage(languageOrdinal) language match { case scc.ThriftLanguage.Unknown => None case _ => Some(language) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/MissingKeyException.scala ================================================ package com.twitter.home_mixer.util object MissingKeyException extends Exception("Missing key") { override def toString: String = getMessage } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/NaviScorerStatsHandler.scala ================================================ package com.twitter.home_mixer.util import com.twitter.finagle.stats.BroadcastStatsReceiver import com.twitter.finagle.stats.Counter import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.stats.Stat import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.HomeFeatures.NaviClientConfigFeature import com.twitter.home_mixer.model.PredictedScoreFeature import com.twitter.home_mixer.model.PredictedScoreFeature.PredictedScoreFeatures import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelIdParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ModelNameParam import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.util.Memoize case class ModelStats(broadcastStatsReceiver: StatsReceiver) { private val StatsReadabilityMultiplier = 1000 private val PredictedScoreStatName = f"predictedScore${StatsReadabilityMultiplier}x" private val MissingScoreStatName = "missingScore" private val ValidScoreStatName = "validScore" private val NullStat = NullStatsReceiver.stat("NullStat") private val NullCounter = NullStatsReceiver.counter("NullCounter") val failuresStat: Stat = broadcastStatsReceiver.stat("failures") val responsesStat: Stat = broadcastStatsReceiver.stat("responses") val invalidResponsesCounter: Counter = broadcastStatsReceiver.counter("invalidResponses") val scoreStat: Stat = broadcastStatsReceiver.stat(f"score${StatsReadabilityMultiplier}x") val negativeFilterCounter: Counter = broadcastStatsReceiver.counter("negativeFiltered") private val predictedScoreStats: Map[PredictedScoreFeature, Stat] = PredictedScoreFeatures.map { scoreFeature => (scoreFeature, broadcastStatsReceiver.stat(scoreFeature.statName, PredictedScoreStatName)) }.toMap private val validScoreCounters: Map[PredictedScoreFeature, Counter] = PredictedScoreFeatures.map { scoreFeature => (scoreFeature, broadcastStatsReceiver.counter(scoreFeature.statName, ValidScoreStatName)) }.toMap private val missingScoreCounters: Map[PredictedScoreFeature, Counter] = PredictedScoreFeatures.map { scoreFeature => (scoreFeature, broadcastStatsReceiver.counter(scoreFeature.statName, MissingScoreStatName)) }.toMap def getPredictedScoreStat(scoreFeature: PredictedScoreFeature): Stat = predictedScoreStats.getOrElse(scoreFeature, NullStat) def getValidScoreCounter(scoreFeature: PredictedScoreFeature): Counter = validScoreCounters.getOrElse(scoreFeature, NullCounter) def getMissingScoreCounter(scoreFeature: PredictedScoreFeature): Counter = missingScoreCounters.getOrElse(scoreFeature, NullCounter) def trackPredictedScoreStats( predictedScoreFeature: PredictedScoreFeature, predictedScoreOpt: Option[Double] ): Unit = { predictedScoreOpt match { case Some(predictedScore) => getPredictedScoreStat(predictedScoreFeature) .add((predictedScore * StatsReadabilityMultiplier).toFloat) getValidScoreCounter(predictedScoreFeature).incr() case None => getMissingScoreCounter(predictedScoreFeature).incr() } } } class NaviScorerStatsHandler(statsReceiver: StatsReceiver, scope: String) { private val scopedStatsReceiver = statsReceiver.scope(scope) // Memoize stats object so they are not created per request private val statsPerModel = Memoize[(String, String, String, String), ModelStats] { case (product, modelId, modelName, clientId) => // Collect stats overall, per product, per model, and per client val broadcastStatsReceiver: StatsReceiver = BroadcastStatsReceiver( Seq( scopedStatsReceiver, scopedStatsReceiver.scope(product), scopedStatsReceiver.scope(modelId).scope(product), scopedStatsReceiver.scope(modelName).scope(product), scopedStatsReceiver.scope(clientId).scope(product) ) ) ModelStats(broadcastStatsReceiver) } /** Retrieve ModelStats for the given query */ def getModelStats( query: PipelineQuery ): ModelStats = { val modelId = query.params(ModelIdParam) val modelName = query.params(ModelNameParam) val modelNameStr = if (modelName.nonEmpty) modelName else "EMPTY_MODEL_NAME" val naviClientConfig = query.features.map(_.get(NaviClientConfigFeature)).get // Should always be present val clientId = query.clientContext.appId.getOrElse(0L).toString statsPerModel( ( query.product.identifier.toString, modelId + naviClientConfig.clusterStr, modelNameStr + naviClientConfig.clusterStr, clientId)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/ObservedKeyValueResultHandler.scala ================================================ package com.twitter.home_mixer.util import com.twitter.finagle.stats.StatsReceiver import com.twitter.servo.keyvalue.KeyValueResult import com.twitter.util.Return import com.twitter.util.Throw import com.twitter.util.Try trait ObservedKeyValueResultHandler { val statsReceiver: StatsReceiver val statScope: String private lazy val scopedStatsReceiver = statsReceiver.scope(statScope) private lazy val keyTotalCounter = scopedStatsReceiver.counter("key/total") private lazy val keyFoundCounter = scopedStatsReceiver.counter("key/found") private lazy val keyNotFoundCounter = scopedStatsReceiver.counter("key/notFound") private lazy val keyFailureCounter = scopedStatsReceiver.counter("key/failure") def observedGet[K, V]( key: Option[K], keyValueResult: KeyValueResult[K, V], ): Try[Option[V]] = { if (key.nonEmpty) { keyTotalCounter.incr() keyValueResult(key.get) match { case Return(Some(value)) => keyFoundCounter.incr() Return(Some(value)) case Return(None) => keyNotFoundCounter.incr() Return(None) case Throw(exception) => keyFailureCounter.incr() Throw(exception) case _ => // never reaches here Return(None) } } else { Throw(MissingKeyException) } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/PhoenixScorerStatsHandler.scala ================================================ package com.twitter.home_mixer.util import com.twitter.finagle.stats.BroadcastStatsReceiver import com.twitter.finagle.stats.Counter import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.finagle.stats.Stat import com.twitter.finagle.stats.StatsReceiver import com.twitter.home_mixer.model.PhoenixPredictedScoreFeature import com.twitter.home_mixer.model.PhoenixPredictedScoreFeature.PhoenixPredictedScoreFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.util.Memoize case class PhoenixModelStats(broadcastStatsReceiver: StatsReceiver) { private val StatsReadabilityMultiplier = 1000 private val PredictedScoreStatName = f"predictedScore${StatsReadabilityMultiplier}x" private val MissingScoreStatName = "missingScore" private val ValidScoreStatName = "validScore" private val NullStat = NullStatsReceiver.stat("NullStat") private val NullCounter = NullStatsReceiver.counter("NullCounter") val failuresStat: Stat = broadcastStatsReceiver.stat("failures") val responsesStat: Stat = broadcastStatsReceiver.stat("responses") val invalidResponsesCounter: Counter = broadcastStatsReceiver.counter("invalidResponses") val scoreStat: Stat = broadcastStatsReceiver.stat(f"score${StatsReadabilityMultiplier}x") val negativeFilterCounter: Counter = broadcastStatsReceiver.counter("negativeFiltered") private val predictedScoreStats: Map[PhoenixPredictedScoreFeature, Stat] = PhoenixPredictedScoreFeatures.map { scoreFeature => (scoreFeature, broadcastStatsReceiver.stat(scoreFeature.featureName, PredictedScoreStatName)) }.toMap private val validScoreCounters: Map[PhoenixPredictedScoreFeature, Counter] = PhoenixPredictedScoreFeatures.map { scoreFeature => (scoreFeature, broadcastStatsReceiver.counter(scoreFeature.featureName, ValidScoreStatName)) }.toMap private val missingScoreCounters: Map[PhoenixPredictedScoreFeature, Counter] = PhoenixPredictedScoreFeatures.map { scoreFeature => (scoreFeature, broadcastStatsReceiver.counter(scoreFeature.featureName, MissingScoreStatName)) }.toMap def getPredictedScoreStat(scoreFeature: PhoenixPredictedScoreFeature): Stat = predictedScoreStats.getOrElse(scoreFeature, NullStat) def getValidScoreCounter(scoreFeature: PhoenixPredictedScoreFeature): Counter = validScoreCounters.getOrElse(scoreFeature, NullCounter) def getMissingScoreCounter(scoreFeature: PhoenixPredictedScoreFeature): Counter = missingScoreCounters.getOrElse(scoreFeature, NullCounter) def trackPredictedScoreStats( predictedScoreFeature: PhoenixPredictedScoreFeature, predictedScoreOpt: Option[Double] ): Unit = { predictedScoreOpt match { case Some(predictedScore) => getPredictedScoreStat(predictedScoreFeature) .add((predictedScore * StatsReadabilityMultiplier).toFloat) getValidScoreCounter(predictedScoreFeature).incr() case None => getMissingScoreCounter(predictedScoreFeature).incr() } } } class PhoenixScorerStatsHandler(statsReceiver: StatsReceiver, scope: String) { private val scopedStatsReceiver = statsReceiver.scope(scope) // Memoize stats object so they are not created per request private val statsPerModel = Memoize[(String, String, String), PhoenixModelStats] { case (product, clientId, cluster) => // Collect stats overall, per product, and per client val broadcastStatsReceiver: StatsReceiver = BroadcastStatsReceiver( Seq( scopedStatsReceiver, scopedStatsReceiver.scope(product), scopedStatsReceiver.scope(product).scope(cluster), scopedStatsReceiver.scope(clientId).scope(product) ) ) PhoenixModelStats(broadcastStatsReceiver) } def getModelStats(query: PipelineQuery, cluster: String): PhoenixModelStats = { val clientId = query.clientContext.appId.getOrElse(0L).toString statsPerModel((query.product.identifier.toString, clientId, cluster)) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/PhoenixUtils.scala ================================================ package com.twitter.home_mixer.util import com.twitter.finagle.grpc.FutureConverters import com.twitter.finagle.service.RetryPolicy import com.twitter.finagle.stats.Stat import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.UserActionsFeature import com.twitter.inject.utils.RetryUtils import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.servo.util.MemoizingStatsReceiver import com.twitter.stitch.Stitch import com.twitter.util.Future import com.twitter.util.Return import com.twitter.util.Throw import com.twitter.util.Try import com.x.user_action_sequence.ActionName import com.x.user_action_sequence.CandidateSet import com.x.user_action_sequence.PredictNextActionsRequest import com.x.user_action_sequence.PredictNextActionsResponse import com.x.user_action_sequence.RecsysPredictorGrpc import com.x.user_action_sequence.TweetBoolFeatures import com.x.user_action_sequence.TweetInfo import io.grpc.ManagedChannel import java.util.concurrent.TimeUnit import scala.collection.JavaConverters._ import scala.util.Random object PhoenixUtils { private val TopLogProbsNum = 50 private val MaxCandidates = 1400 def getTweetInfoFromCandidates( candidateIds: Seq[TweetCandidate], featureMaps: Seq[FeatureMap] ): Seq[TweetInfo] = { assert( candidateIds.length == featureMaps.length, "FeatureMap doesn't match candidateIds in length") val candidateWithFeatureMaps = candidateIds.zip(featureMaps) candidateWithFeatureMaps.take(MaxCandidates).map { case (candidate, featureMap) => val tweetBooleanFeatureBuilder = TweetBoolFeatures.newBuilder() val (sourceTweetId, sourceAuthorId) = ( featureMap.getOrElse(SourceTweetIdFeature, None), featureMap.getOrElse(SourceUserIdFeature, None)) match { case (Some(sourceTweetId), Some(sourceAuthorId)) => tweetBooleanFeatureBuilder.setIsRetweet(true) (sourceTweetId, sourceAuthorId) case _ => tweetBooleanFeatureBuilder.setIsRetweet(false) (candidate.id, featureMap.get(AuthorIdFeature).get) } tweetBooleanFeatureBuilder .setIsForYouPage(true) .setIsPromotedTweet(false) .setIsReply(featureMap.getOrElse(InReplyToTweetIdFeature, None).isDefined) .setIsQuote(featureMap.getOrElse(QuotedTweetIdFeature, None).isDefined) TweetInfo .newBuilder() .setTweetId(sourceTweetId) .setAuthorId(sourceAuthorId) .setTweetBoolFeatures(tweetBooleanFeatureBuilder.build()) .build() } } def createCandidateSets( query: PipelineQuery, tweetInfos: Seq[TweetInfo] ): PredictNextActionsRequest = { val actionsSeqOpt = query.features.flatMap(_.getOrElse(UserActionsFeature, None)) val candidateSets = CandidateSet .newBuilder() .setUserId(query.getRequiredUserId) .addAllCandidates(tweetInfos.asJava) .build() val predictionRequestBuilder = PredictNextActionsRequest .newBuilder() .addCandidateSets(candidateSets) .setReturnLogprob(true) .setTopLogprobsNum(TopLogProbsNum) actionsSeqOpt.foreach { actionsSeq => predictionRequestBuilder.addSequences(actionsSeq) } predictionRequestBuilder.build() } def predict( request: PredictNextActionsRequest, channels: Seq[ManagedChannel], cluster: String, timeoutMs: Int, baseStat: MemoizingStatsReceiver, ): Stitch[PredictNextActionsResponse] = { val timeStat = baseStat.scope("rpcTime") val successStat = baseStat.scope("rpcSuccess") val failureStat = baseStat.scope("rpcFailure") val failureMsgStat = baseStat.scope("rpcFailureMsg") def attemptPredict(): Future[PredictNextActionsResponse] = { // This is kept inside predict so that it doesn't use the same channel for retries val channel = channels(Random.nextInt(channels.length)) val recsysPredictorFutureStub = RecsysPredictorGrpc .newFutureStub(channel).withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS) FutureConverters .RichListenableFuture(recsysPredictorFutureStub.predictNextActions(request)) .toTwitter } // Retry configuration: 2 attempts, 500ms per attempt, total 1s max val retryPolicy = RetryPolicy .tries[Try[PredictNextActionsResponse]](2, { case Throw(_) => true; case Return(_) => false }) val attemptPredictRetriedFuture = RetryUtils.retryFuture[PredictNextActionsResponse](retryPolicy)(attemptPredict) Stitch .callFuture(Stat.timeFuture(timeStat.stat(cluster))(attemptPredictRetriedFuture)) .onFailure { e => val errMsg = e.getMessage.replaceAll("\\W", "").takeRight(100) failureStat.counter(cluster).incr() failureMsgStat.scope(cluster).counter(errMsg).incr() }.onSuccess { _ => successStat.counter(cluster).incr() } } def getPredictionResponseMap( request: PredictNextActionsRequest, channels: Seq[ManagedChannel], cluster: String, timeoutMs: Int, memoizingStatsReceiver: MemoizingStatsReceiver ): Stitch[Map[Long, Map[ActionName, Double]]] = { val predictionsResponse = predict(request, channels, cluster, timeoutMs, memoizingStatsReceiver) predictionsResponse.map { predictions => val distributionSetsList = predictions.getDistributionSetsList val candidateDistributions = if (!distributionSetsList.isEmpty) { distributionSetsList.get(0).getCandidateDistributionsList.asScala } else Seq.empty candidateDistributions.map { distribution => val candidateId = distribution.getCandidate.getTweetId val probMap = distribution.getTopLogProbsList.asScala.zipWithIndex.map { case (logProb, idx) => ActionName.forNumber(idx) -> math.exp(logProb.doubleValue()) }.toMap candidateId -> probMap }.toMap } } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/ReplyRetweetUtil.scala ================================================ package com.twitter.home_mixer.util import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures object ReplyRetweetUtil { def isEligibleReply(candidate: CandidateWithFeatures[TweetCandidate]): Boolean = { candidate.features.getOrElse(InReplyToTweetIdFeature, None).nonEmpty && !candidate.features.getOrElse(IsRetweetFeature, false) } /** * Builds a map from reply tweet to all ancestors that are also hydrated candidates. If a reply * does not have any ancestors which are also candidates, it will not add to the returned Map. * Make sure ancestors are bottom-up ordered such that: * (1) if parent tweet is a candidate, it should be the first item at the returned ancestors; * (2) if root tweet is a candidate, it should be the last item at the returned ancestors. * Retweets of replies or replies to retweets are not included. */ def replyToAncestorTweetCandidatesMap( candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Map[Long, Seq[CandidateWithFeatures[TweetCandidate]]] = { val replyToAncestorTweetIdsMap: Map[Long, Seq[Long]] = candidates.flatMap { candidate => if (isEligibleReply(candidate)) { val ancestorIds = if (candidate.features.getOrElse(AncestorsFeature, Seq.empty).nonEmpty) { candidate.features.getOrElse(AncestorsFeature, Seq.empty).map(_.tweetId) } else { Seq( candidate.features.getOrElse(InReplyToTweetIdFeature, None), candidate.features.getOrElse(ConversationModuleIdFeature, None) ).flatten.distinct } Some(candidate.candidate.id -> ancestorIds) } else { None } }.toMap val ancestorTweetIds = replyToAncestorTweetIdsMap.values.flatten.toSet val ancestorTweetsMapById: Map[Long, CandidateWithFeatures[TweetCandidate]] = candidates .filter { maybeAncestor => ancestorTweetIds.contains(maybeAncestor.candidate.id) }.map { ancestor => ancestor.candidate.id -> ancestor }.toMap replyToAncestorTweetIdsMap .mapValues { ancestorTweetIds => ancestorTweetIds.flatMap { ancestorTweetId => ancestorTweetsMapById.get(ancestorTweetId) } }.filter { case (reply, ancestors) => ancestors.nonEmpty } } /** * This map is the opposite of [[replyToAncestorTweetCandidatesMap]]. * Builds a map from ancestor tweet to all descendant replies that are also hydrated candidates. * Currently, we only return two ancestors at most: one is inReplyToTweetId and the other * is conversationId. * Retweets of replies are not included. */ def ancestorTweetIdToDescendantRepliesMap( candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Map[Long, Seq[CandidateWithFeatures[TweetCandidate]]] = { val tweetToCandidateMap = candidates.map(c => c.candidate.id -> c).toMap replyToAncestorTweetCandidatesMap(candidates).toSeq .flatMap { case (reply, ancestorTweets) => ancestorTweets.map { ancestor => (ancestor.candidate.id, reply) } }.groupBy { case (ancestor, reply) => ancestor } .mapValues { ancestorReplyPairs => ancestorReplyPairs.map(_._2).distinct }.mapValues(tweetIds => tweetIds.map(tid => tweetToCandidateMap(tid))) } /** * Builds a map from reply tweet to inReplyToTweet which is also a candidate. * Retweets of replies or replies to retweets are not included */ def replyTweetIdToInReplyToTweetMap( candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Map[Long, CandidateWithFeatures[TweetCandidate]] = { val eligibleReplyCandidates = candidates.filter { candidate => isEligibleReply(candidate) && candidate.features .getOrElse(InReplyToTweetIdFeature, None) .nonEmpty } val inReplyToTweetIds = eligibleReplyCandidates .flatMap(_.features.getOrElse(InReplyToTweetIdFeature, None)) .toSet val inReplyToTweetIdToTweetMap: Map[Long, CandidateWithFeatures[TweetCandidate]] = candidates .filter { maybeInReplyToTweet => inReplyToTweetIds.contains(maybeInReplyToTweet.candidate.id) }.map { inReplyToTweet => inReplyToTweet.candidate.id -> inReplyToTweet }.toMap eligibleReplyCandidates.flatMap { reply => val inReplyToTweetId = reply.features.getOrElse(InReplyToTweetIdFeature, None) if (inReplyToTweetId.nonEmpty) { inReplyToTweetIdToTweetMap.get(inReplyToTweetId.get).map { inReplyToTweet => reply.candidate.id -> inReplyToTweet } } else { None } }.toMap } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/RerankerUtil.scala ================================================ package com.twitter.home_mixer.util import com.twitter.finagle.stats.Counter import com.twitter.home_mixer.model.PredictedScoreFeature.PredictedScoreFeatures import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.ConstantNegativeHead import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.EnableNegSectionRankingParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.NegativeScoreConstantFilterThresholdParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.NegativeScoreNormFilterThresholdParam import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.NormalizedNegativeHead import com.twitter.home_mixer.param.HomeGlobalParams.Scoring.UseWeightForNegHeadParam import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.pipeline.PipelineQuery object RerankerUtil { val Epsilon = 0.001 def computeModelScores( query: PipelineQuery, candidate: CandidateWithFeatures[TweetCandidate], modelStatsOpt: Option[ModelStats] = None ): Seq[(Double, Double)] = { PredictedScoreFeatures.map { predictedScoreFeature => val predictedScoreOpt = predictedScoreFeature.extractScore(candidate.features, query) modelStatsOpt.foreach(_.trackPredictedScoreStats(predictedScoreFeature, predictedScoreOpt)) val weight = query.features.flatMap(_.get(predictedScoreFeature.weightQueryFeature)).getOrElse(0.0) val bias = predictedScoreFeature.biasQueryFeature .flatMap(feature => query.features.flatMap(_.get(feature))).getOrElse(0.0) val score = if (predictedScoreFeature.isEligible(candidate.features, query)) predictedScoreOpt.getOrElse(0.0) else bias (score, weight) } } def getScoresWithPerHeadMax( scoresAndWeightsSeq: Seq[Seq[(Double, Double)]] ): Seq[Seq[(Double, Double, Double)]] = { if (scoresAndWeightsSeq.isEmpty) Seq.empty else { // Step 1: Transpose scores to group by heads val headsScores: Seq[Seq[Double]] = scoresAndWeightsSeq.transpose.map { headScores => headScores.map { case (score, _) => score } } // Step 2: Get max scores per head val headMaxScores: Seq[Double] = headsScores.map(_.max).toIndexedSeq // Step 3: Use max scores to get per head transformed scores scoresAndWeightsSeq.map { candidateScores => candidateScores.zipWithIndex.map { case ((score, weight), headIdx) => val headMaxScore = headMaxScores(headIdx) (score, headMaxScore, weight) } } } } def computeDebugMetadata( debugStr: String, featureNames: Seq[String], transformedScores: Seq[(Double, Double, Double)], finalScore: Double ): String = { assert( featureNames.size == transformedScores.size, "Feature names size doesn't matter scores size") val contributions: Seq[(String, Double)] = featureNames .zip(transformedScores) .collect { case (feature, (score, _, weight)) if weight >= 0 => (feature, score * weight) } val topContributors: Seq[String] = contributions.collect { case (name, contrib) if finalScore > 0 && (contrib / finalScore) > 0.3 => f"$name:%%.2f".format(contrib / finalScore) } debugStr + s" [${topContributors.mkString(", ")}]" } def aggregateWeightedScores( query: PipelineQuery, scoresAndWeights: Seq[(Double, Double, Double)], negativeFilterCounter: Counter ): Double = { val thresholdNegative = query.params(NegativeScoreConstantFilterThresholdParam) val thresholdNegativeNormalized = query.params(NegativeScoreNormFilterThresholdParam) val enableNegNormalized = query.params(NormalizedNegativeHead) val enableNegConstant = query.params(ConstantNegativeHead) val useWeightForNeg = query.params(UseWeightForNegHeadParam) val negSectionRanking = query.params(EnableNegSectionRankingParam) val (_, maxHeadScores, modelWeights) = scoresAndWeights.unzip3 val combinedScoreSum: Double = { scoresAndWeights.foldLeft(0.0) { case (combinedScore, (score, maxHeadScore, weight)) => if (weight >= 0.0 || useWeightForNeg) { combinedScore + score * weight } else { // Apply filtering logic only for negative weights val normScore = if (maxHeadScore == 0.0) 0.0 else score / maxHeadScore val negFilterNorm = enableNegNormalized && normScore > thresholdNegativeNormalized val negFilterConstant = enableNegConstant && score > thresholdNegative if (negFilterNorm || negFilterConstant) { negativeFilterCounter.incr() // This should be shipped and cleaned as soon as possible if (negSectionRanking) { // Assumes negative scores are not greater than 0.9 and clipped to 1 combinedScore + weight * (1.0 min (score + 0.1)) } else combinedScore + weight } else combinedScore } } } val positiveModelWeightsSum = modelWeights.filter(_ > 0.0).sum val negativeModelWeightsSum = modelWeights.filter(_ < 0).sum.abs val modelWeightsSum = positiveModelWeightsSum + negativeModelWeightsSum val weightedScoresSum = if (modelWeightsSum == 0) combinedScoreSum.max(0.0) else if (combinedScoreSum < 0) (combinedScoreSum + negativeModelWeightsSum) / modelWeightsSum * Epsilon else combinedScoreSum + Epsilon weightedScoresSum } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/SignalUtil.scala ================================================ package com.twitter.home_mixer.util import com.twitter.home_mixer.model.HomeFeatures.LowSignalUserFeature import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.usersignalservice.{thriftscala => uss} object SignalUtil { val ExplicitSignals: Seq[uss.SignalType] = Seq( uss.SignalType.TweetFavorite, uss.SignalType.Retweet, uss.SignalType.Reply, uss.SignalType.TweetBookmarkV1, uss.SignalType.TweetShareV1 ) private val SmallFollowGraphSize = 5 def isLowSignalUser(query: PipelineQuery): Boolean = { val followGraphSize = query.features.map(_.getOrElse(SGSFollowedUsersFeature, Seq.empty).size) val smallFollowGraph = followGraphSize.exists(_ < SmallFollowGraphSize) val lowSignal = query.features.map(_.getOrElse(LowSignalUserFeature, false)).getOrElse(false) lowSignal && smallFollowGraph } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/TensorFlowUtil.scala ================================================ package com.twitter.home_mixer.util import com.twitter.ml.api.thriftscala.FloatTensor import com.twitter.ml.api.util.BufferToIterators.RichFloatBuffer import java.nio.ByteBuffer import java.nio.ByteOrder /** * Contains functionality to transform data records and Tensors */ object TensorFlowUtil { private def skipEmbeddingBBHeader(bb: ByteBuffer): ByteBuffer = { val bb_copy = bb.duplicate() bb_copy.getLong() bb_copy } private def byteBufferToFloatIterator( bb: ByteBuffer ): Iterator[Float] = { bb.order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer.iterator } def embeddingByteBufferToFloatTensor( bb: ByteBuffer ): FloatTensor = { val bb_content = skipEmbeddingBBHeader(bb) FloatTensor(byteBufferToFloatIterator(bb_content).map(_.toDouble).toList) } def embeddingNoHeaderByteBufferToFloatTensor( bb: ByteBuffer ): FloatTensor = { FloatTensor(byteBufferToFloatIterator(bb).map(_.toDouble).toList) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/TweetImpressionsHelper.scala ================================================ package com.twitter.home_mixer.util import com.twitter.home_mixer.model.HomeFeatures.TweetImpressionsFeature import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweets import com.twitter.product_mixer.core.feature.featuremap.FeatureMap object TweetImpressionsHelper { def tweetImpressions(features: FeatureMap): Set[Long] = { val manhattanImpressions = features.getOrElse(TweetImpressionsFeature, Seq.empty).flatMap(_.tweetIds) val memcacheImpressions = features.getOrElse(ImpressedTweets, Seq.empty) (manhattanImpressions ++ memcacheImpressions).toSet } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/UrtUtil.scala ================================================ package com.twitter.home_mixer.util import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.DeepLink import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ExternalUrl import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Url import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.UrtEndpoint import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.UrtEndpointOptions import com.twitter.timelines.render.{thriftscala => t} object UrtUtil { def transformUrl(url: t.Url): Url = { val endpointOptions = url.urtEndpointOptions.map { options => UrtEndpointOptions( requestParams = options.requestParams.map(_.toMap), title = options.title, cacheId = options.cacheId, subtitle = options.subtitle ) } val urlType = url.urlType match { case t.UrlType.ExternalUrl => ExternalUrl case t.UrlType.DeepLink => DeepLink case t.UrlType.UrtEndpoint => UrtEndpoint case t.UrlType.EnumUnknownUrlType(field) => throw new UnknownUrlTypeException(field) } Url(urlType = urlType, url = url.url, urtEndpointOptions = endpointOptions) } } class UnknownUrlTypeException(field: Int) extends UnsupportedOperationException(s"Unknown url type: $field") ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "src/java/com/twitter/search/common/schema/base", "src/java/com/twitter/search/common/schema/earlybird", "src/java/com/twitter/search/common/util/lang", "src/java/com/twitter/search/queryparser/query:core-query-nodes", "src/thrift/com/twitter/search:earlybird-scala", "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", "timelines/src/main/scala/com/twitter/timelines/earlybird/common/options", "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", "timelines/src/main/scala/com/twitter/timelines/model/types", "timelines/src/main/scala/com/twitter/timelines/util/stats", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdRequestUtil.scala ================================================ package com.twitter.home_mixer.util.earlybird import com.twitter.conversions.DurationOps._ import com.twitter.search.common.query.thriftjava.{thriftscala => scq} import com.twitter.search.common.ranking.{thriftscala => scr} import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.timelines.clients.relevance_search.SearchClient.TweetFeatures import com.twitter.timelines.clients.relevance_search.SearchClient.TweetTypes import com.twitter.timelines.clients.relevance_search.SearchQueryBuilder import com.twitter.timelines.clients.relevance_search.SearchQueryBuilder.QueryWithNamedDisjunctions import com.twitter.timelines.earlybird.common.options.EarlybirdScoringModelConfig import com.twitter.timelines.earlybird.common.utils.SearchOperator import com.twitter.util.Duration object EarlybirdRequestUtil { val DefaultMaxHitsToProcess = 1000 val DefaultSearchProcessingTimeout: Duration = 200.milliseconds val DefaultFeatureHydrationTimeout: Duration = 50.milliseconds val DefaultHydrationMaxNumResultsPerShard = 1000 val DefaultQueryMaxNumResultsPerShard = 300 val DefaultHydrationCollectorParams = mkCollectorParams(DefaultHydrationMaxNumResultsPerShard) private val queryBuilder = new SearchQueryBuilder object EarlybirdScoringModels { val UnifiedEngagementProd: Seq[EarlybirdScoringModelConfig] = Seq( EarlybirdScoringModelConfig("timelines_unified_engagement_prod.schema_based", 1.0) ) val UnifiedEngagementRectweet: Seq[EarlybirdScoringModelConfig] = Seq( EarlybirdScoringModelConfig("timelines_unified_engagement_rectweet.schema_based", 1.0) ) } private[earlybird] def mkCollectorParams(numResultsToReturn: Int): scq.CollectorParams = { scq.CollectorParams( // numResultsToReturn defines how many results each EB shard will return to search root numResultsToReturn = numResultsToReturn, // terminationParams.maxHitsToProcess is used for early terminating per shard results fetching. terminationParams = Some( scq.CollectorTerminationParams( maxHitsToProcess = Some(DefaultMaxHitsToProcess), timeoutMs = DefaultSearchProcessingTimeout.inMilliseconds.toInt )) ) } private def getRankingParams( authorScoreMap: Option[Map[Long, Double]], tensorflowModel: Option[String], ebModels: Seq[EarlybirdScoringModelConfig] ): Option[scr.ThriftRankingParams] = { if (tensorflowModel.nonEmpty) { Some( scr.ThriftRankingParams( `type` = Some(scr.ThriftScoringFunctionType.TensorflowBased), selectedTensorflowModel = tensorflowModel, minScore = -1.0e100, applyBoosts = false, authorSpecificScoreAdjustments = authorScoreMap ) ) } else if (ebModels.nonEmpty) { Some( scr.ThriftRankingParams( `type` = Some(scr.ThriftScoringFunctionType.ModelBased), selectedModels = Some(ebModels.map(m => m.name -> m.weight).toMap), applyBoosts = false, minScore = -1.0e100, authorSpecificScoreAdjustments = authorScoreMap ) ) } else None } def getTweetsRequest( userId: Option[Long], clientId: Option[String], skipVeryRecentTweets: Boolean, queryUserIds: Set[Long], retweetsMutedUserIds: Set[Long], beforeTweetIdExclusive: Option[Long], afterTweetIdExclusive: Option[Long], excludedTweetIds: Option[Set[Long]] = None, maxCount: Int, tweetTypes: TweetTypes.ValueSet, authorScoreMap: Option[Map[Long, Double]] = None, tensorflowModel: Option[String] = None, ebModels: Seq[EarlybirdScoringModelConfig] = Seq.empty, queryMaxNumResultsPerShard: Int = DefaultQueryMaxNumResultsPerShard, enableExcludeSourceTweetIdsQuery: Boolean = false, isVideoOnlyRequest: Boolean = false, isRecency: Boolean = false, getOlderTweets: Boolean = false, ): eb.EarlybirdRequest = { val QueryWithNamedDisjunctions(query, namedDisjunctionMap) = queryBuilder.create( queryUserIds, retweetsMutedUserIds, beforeTweetIdExclusive, afterTweetIdExclusive, semanticCoreIds = None, languages = None, tweetTypes = tweetTypes, searchOperator = SearchOperator.Exclude, tweetFeatures = TweetFeatures.All, excludedTweetIds = excludedTweetIds.getOrElse(Set.empty), enableExcludeSourceTweetIdsQuery = enableExcludeSourceTweetIdsQuery, isVideoOnlyRequest = isVideoOnlyRequest ) val ebRankingParams = getRankingParams(authorScoreMap, tensorflowModel, ebModels) val relOptions = RelevanceSearchUtil.RelevanceOptions.copy( rankingParams = ebRankingParams, returnAllResults = Some(false) ) val metadataOptions = if (isRecency) eb.ThriftSearchResultMetadataOptions( getTweetUrls = false, getResultLocation = false, deprecatedGetTopicIDs = false, getInReplyToStatusId = true, getReferencedTweetAuthorId = true, getFromUserId = true ) else RelevanceSearchUtil.MetadataOptions val queryUserIdsSeq = queryUserIds.toSeq val namedDisjunctionMapOpt = if (namedDisjunctionMap.isEmpty) None else Some(namedDisjunctionMap.mapValues(_.toSeq)) val rankingMode = if (isRecency) eb.ThriftSearchRankingMode.Recency else eb.ThriftSearchRankingMode.Relevance val thriftQuery = eb.ThriftSearchQuery( serializedQuery = Some(query.serialize), fromUserIDFilter64 = Some(queryUserIdsSeq), numResults = maxCount, collectConversationId = true, rankingMode = rankingMode, relevanceOptions = Some(relOptions), collectorParams = Some(mkCollectorParams(queryMaxNumResultsPerShard)), facetFieldNames = Some(RelevanceSearchUtil.FacetsToFetch), resultMetadataOptions = Some(metadataOptions), searcherId = userId, searchStatusIds = None, namedDisjunctionMap = namedDisjunctionMapOpt ) eb.EarlybirdRequest( searchQuery = thriftQuery, clientId = clientId, getOlderResults = Some(getOlderTweets), followedUserIds = Some(queryUserIdsSeq), getProtectedTweetsOnly = Some(false), timeoutMs = DefaultSearchProcessingTimeout.inMilliseconds.toInt, skipVeryRecentTweets = skipVeryRecentTweets, numResultsToReturnAtRoot = Some(maxCount) ) } def getTweetsFeaturesRequest( userId: Option[Long], tweetIds: Option[Seq[Long]], clientId: Option[String], getOnlyProtectedTweets: Boolean = false, authorScoreMap: Option[Map[Long, Double]] = None, tensorflowModel: Option[String] = None, ebModels: Seq[EarlybirdScoringModelConfig] = Seq.empty ): eb.EarlybirdRequest = { val candidateSize = tweetIds.getOrElse(Seq.empty).size val ebRankingParams = getRankingParams(authorScoreMap, tensorflowModel, ebModels) val relOptions = RelevanceSearchUtil.RelevanceOptions.copy( rankingParams = ebRankingParams ) val thriftQuery = eb.ThriftSearchQuery( numResults = candidateSize, collectConversationId = true, rankingMode = eb.ThriftSearchRankingMode.Relevance, relevanceOptions = Some(relOptions), collectorParams = Some(DefaultHydrationCollectorParams), facetFieldNames = Some(RelevanceSearchUtil.FacetsToFetch), resultMetadataOptions = Some(RelevanceSearchUtil.MetadataOptions), searcherId = userId, searchStatusIds = tweetIds.map(_.toSet), ) eb.EarlybirdRequest( searchQuery = thriftQuery, clientId = clientId, getOlderResults = Some(false), getProtectedTweetsOnly = Some(getOnlyProtectedTweets), timeoutMs = DefaultSearchProcessingTimeout.inMilliseconds.toInt, skipVeryRecentTweets = true, // This param decides # of tweets to return from search superRoot and realtime/protected/Archive roots. // It takes higher precedence than ThriftSearchQuery.numResults numResultsToReturnAtRoot = Some(candidateSize), partitionTimeoutMs = Some(DefaultFeatureHydrationTimeout.inMillis.toInt) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/EarlybirdResponseUtil.scala ================================================ package com.twitter.home_mixer.util.earlybird import com.twitter.product_mixer.core.util.OffloadFuturePools import com.twitter.search.common.constants.{thriftscala => scc} import com.twitter.search.common.features.{thriftscala => sc} import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant._ import com.twitter.search.common.util.lang.ThriftLanguageUtil import com.twitter.search.earlybird.{thriftscala => eb} import com.twitter.timelines.earlybird.common.utils.InNetworkEngagement import com.twitter.util.Future object EarlybirdResponseUtil { private[earlybird] val Mentions: String = "mentions" private val CharsToRemoveFromMentions: Set[Char] = "@".toSet // Default value of settings of ThriftTweetFeatures. private[earlybird] val DefaultEarlybirdFeatures: sc.ThriftTweetFeatures = sc.ThriftTweetFeatures() private[earlybird] val DefaultCount = 0 private[earlybird] val DefaultLanguage = 0 private[earlybird] val DefaultScore = 0.0 private[earlybird] val DefaultEBResponseProcessParallelism = 32 def getTweetCountByAuthorId( searchResults: Seq[eb.ThriftSearchResult] ): Map[Long, Int] = { val tweetCounts = scala.collection.mutable.Map.empty[Long, Int] searchResults.foreach { result => val authorId = result.metadata.map(_.fromUserId).getOrElse(0L) tweetCounts(authorId) = tweetCounts.getOrElse(authorId, 0) + 1 } tweetCounts.toMap.withDefaultValue(0) } def getLanguage(uiLanguageCode: Option[String]): Option[scc.ThriftLanguage] = { uiLanguageCode.flatMap { languageCode => scc.ThriftLanguage.get(ThriftLanguageUtil.getThriftLanguageOf(languageCode).getValue) } } private def getMentions(result: eb.ThriftSearchResult): Seq[String] = { val facetLabels = result.metadata.flatMap(_.facetLabels).getOrElse(Seq.empty) getFacets(facetLabels, Mentions, CharsToRemoveFromMentions) } private def getFacets( facetLabels: Seq[eb.ThriftFacetLabel], facetName: String, charsToRemove: Set[Char] ): Seq[String] = { facetLabels.filter(_.fieldName == facetName).map(_.label.filterNot(charsToRemove)) } private def isUserMentioned( screenName: Option[String], mentions: Seq[String], mentionsInSourceTweet: Seq[String] ): Boolean = isUserMentioned(screenName, mentions) || isUserMentioned(screenName, mentionsInSourceTweet) private def isUserMentioned( screenName: Option[String], mentions: Seq[String] ): Boolean = { screenName .exists { screenName => mentions.exists(_.equalsIgnoreCase(screenName)) } } private[earlybird] def isUsersMainLanguage( tweetLanguage: scc.ThriftLanguage, userLanguages: Seq[scc.ThriftLanguage] ): Boolean = { (tweetLanguage != scc.ThriftLanguage.Unknown) && userLanguages.headOption.contains( tweetLanguage) } private[earlybird] def isUsersLanguage( tweetLanguage: scc.ThriftLanguage, userLanguages: Seq[scc.ThriftLanguage] ): Boolean = { (tweetLanguage != scc.ThriftLanguage.Unknown) && userLanguages.contains(tweetLanguage) } private[earlybird] def isUILanguage( tweetLanguage: scc.ThriftLanguage, uiLanguage: Option[scc.ThriftLanguage] ): Boolean = { (tweetLanguage != scc.ThriftLanguage.Unknown) && uiLanguage.contains(tweetLanguage) } private def getBooleanOptFeature( featureName: EarlybirdFieldConstant, resultMapOpt: Option[scala.collection.Map[Int, Boolean]], defaultValue: Boolean = false, ): Option[Boolean] = { resultMapOpt.map { _.getOrElse(featureName.getFieldId, defaultValue) } } private def getDoubleAsIntOptFeature( featureName: EarlybirdFieldConstant, resultMapOpt: Option[scala.collection.Map[Int, Double]] ): Option[Int] = { if (resultMapOpt.exists(_.contains(featureName.getFieldId))) resultMapOpt .map { _.get(featureName.getFieldId) } .flatMap { doubleValue => doubleValue.map(_.toInt) } else None } private def getIntOptFeature( featureName: EarlybirdFieldConstant, resultMapOpt: Option[scala.collection.Map[Int, Int]] ): Option[Int] = { if (resultMapOpt.exists(_.contains(featureName.getFieldId))) resultMapOpt.flatMap { _.get(featureName.getFieldId) } else None } def getTweetThriftFeaturesByTweetId( searcherUserId: Long, screenName: Option[String], userLanguages: Seq[scc.ThriftLanguage], uiLanguageCode: Option[String] = None, followedUserIds: Set[Long], mutuallyFollowingUserIds: Set[Long], idToSearchResults: Map[Long, eb.ThriftSearchResult] ): Future[Map[Long, sc.ThriftTweetFeatures]] = { val searchResults = idToSearchResults.values.toSeq val inNetworkEngagement = InNetworkEngagement(followedUserIds.toSeq, mutuallyFollowingUserIds, searchResults) val tweetCountByAuthorId = getTweetCountByAuthorId(searchResults) val uiLanguage = getLanguage(uiLanguageCode) val idWithFeaturesSeqFu = OffloadFuturePools.parallelize[ eb.ThriftSearchResult, (Long, sc.ThriftTweetFeatures) ]( inputSeq = searchResults, transformer = (searchResult: eb.ThriftSearchResult) => ( searchResult.id, getThriftTweetFeaturesFromSearchResult( searcherUserId, screenName, userLanguages, uiLanguage, tweetCountByAuthorId, followedUserIds, mutuallyFollowingUserIds, idToSearchResults, inNetworkEngagement, searchResult )), parallelism = DefaultEBResponseProcessParallelism, ) idWithFeaturesSeqFu.map(idWithFeaturesSeq => idWithFeaturesSeq.map(idWithFeatures => idWithFeatures._1 -> idWithFeatures._2).toMap) } def getThriftTweetFeaturesFromSearchResult( searcherUserId: Long, screenName: Option[String], userLanguages: Seq[scc.ThriftLanguage], uiLanguage: Option[scc.ThriftLanguage], tweetCountByAuthorId: Map[Long, Int], followedUserIds: Set[Long], mutuallyFollowingUserIds: Set[Long], idToSearchResults: Map[Long, eb.ThriftSearchResult], inNetworkEngagement: InNetworkEngagement, searchResult: eb.ThriftSearchResult, ): sc.ThriftTweetFeatures = { val applyFeatures = (applyUserIndependentFeatures( searchResult )(_)).andThen( applyUserDependentFeatures( searcherUserId, screenName, userLanguages, uiLanguage, tweetCountByAuthorId, followedUserIds, mutuallyFollowingUserIds, idToSearchResults, inNetworkEngagement, searchResult )(_) ) val tweetFeatures = searchResult.tweetFeatures.getOrElse(DefaultEarlybirdFeatures) applyFeatures(tweetFeatures) } private[earlybird] def applyUserIndependentFeatures( result: eb.ThriftSearchResult )( thriftTweetFeatures: sc.ThriftTweetFeatures ): sc.ThriftTweetFeatures = { val features = result.metadata .map { metadata => val isRetweet = metadata.isRetweet.getOrElse(false) val isReply = metadata.isReply.getOrElse(false) val searchResultSchemaFeatures = metadata.extraMetadata.flatMap(_.features) val booleanSearchResultSchemaFeatures = searchResultSchemaFeatures.flatMap(_.boolValues) val intSearchResultSchemaFeatures = searchResultSchemaFeatures.flatMap(_.intValues) val doubleSearchResultSchemaFeatures = searchResultSchemaFeatures.flatMap(_.doubleValues) thriftTweetFeatures.copy( // Info about the Tweet. isRetweet = isRetweet, isOffensive = metadata.isOffensive.getOrElse(false), isReply = isReply, fromVerifiedAccount = metadata.fromVerifiedAccount.getOrElse(false), cardType = metadata.cardType, signature = metadata.signature, language = metadata.language, isAuthorNSFW = metadata.isUserNSFW.getOrElse(false), isAuthorBot = metadata.isUserBot.getOrElse(false), isAuthorSpam = metadata.isUserSpam.getOrElse(false), isSensitiveContent = metadata.extraMetadata.flatMap(_.isSensitiveContent).getOrElse(false), isAuthorProfileEgg = metadata.extraMetadata.flatMap(_.profileIsEggFlag).getOrElse(false), isAuthorNew = metadata.extraMetadata.flatMap(_.isUserNewFlag).getOrElse(false), linkLanguage = metadata.extraMetadata.flatMap(_.linkLanguage).getOrElse(DefaultLanguage), // Info about Tweet content/media. hasCard = metadata.hasCard.getOrElse(false), hasImage = metadata.hasImage.getOrElse(false), hasNews = metadata.hasNews.getOrElse(false), hasVideo = metadata.hasVideo.getOrElse(false), hasConsumerVideo = metadata.hasConsumerVideo.getOrElse(false), hasProVideo = metadata.hasProVideo.getOrElse(false), hasVine = metadata.hasVine.getOrElse(false), hasPeriscope = metadata.hasPeriscope.getOrElse(false), hasNativeVideo = metadata.hasNativeVideo.getOrElse(false), hasNativeImage = metadata.hasNativeImage.getOrElse(false), hasLink = metadata.hasLink.getOrElse(false), hasVisibleLink = metadata.hasVisibleLink.getOrElse(false), hasTrend = metadata.hasTrend.getOrElse(false), hasMultipleHashtagsOrTrends = metadata.hasMultipleHashtagsOrTrends.getOrElse(false), hasQuote = metadata.extraMetadata.flatMap(_.hasQuote), urlsList = metadata.tweetUrls.map { _.map(_.originalUrl) }, hasMultipleMedia = metadata.extraMetadata.flatMap(_.hasMultipleMediaFlag).getOrElse(false), visibleTokenRatio = getIntOptFeature(VISIBLE_TOKEN_RATIO, intSearchResultSchemaFeatures), // Various counts. favCount = metadata.favCount.getOrElse(DefaultCount), replyCount = metadata.replyCount.getOrElse(DefaultCount), retweetCount = metadata.retweetCount.getOrElse(DefaultCount), quoteCount = metadata.extraMetadata.flatMap(_.quotedCount), embedsImpressionCount = metadata.embedsImpressionCount.getOrElse(DefaultCount), embedsUrlCount = metadata.embedsUrlCount.getOrElse(DefaultCount), videoViewCount = metadata.videoViewCount.getOrElse(DefaultCount), numMentions = metadata.extraMetadata.flatMap(_.numMentions).getOrElse(DefaultCount), numHashtags = metadata.extraMetadata.flatMap(_.numHashtags).getOrElse(DefaultCount), favCountV2 = metadata.extraMetadata.flatMap(_.favCountV2), replyCountV2 = metadata.extraMetadata.flatMap(_.replyCountV2), retweetCountV2 = metadata.extraMetadata.flatMap(_.retweetCountV2), weightedFavoriteCount = metadata.extraMetadata.flatMap(_.weightedFavCount), weightedReplyCount = metadata.extraMetadata.flatMap(_.weightedReplyCount), weightedRetweetCount = metadata.extraMetadata.flatMap(_.weightedRetweetCount), weightedQuoteCount = metadata.extraMetadata.flatMap(_.weightedQuoteCount), embedsImpressionCountV2 = getDoubleAsIntOptFeature(EMBEDS_IMPRESSION_COUNT_V2, doubleSearchResultSchemaFeatures), embedsUrlCountV2 = getDoubleAsIntOptFeature(EMBEDS_URL_COUNT_V2, doubleSearchResultSchemaFeatures), decayedFavoriteCount = getDoubleAsIntOptFeature(DECAYED_FAVORITE_COUNT, doubleSearchResultSchemaFeatures), decayedRetweetCount = getDoubleAsIntOptFeature(DECAYED_RETWEET_COUNT, doubleSearchResultSchemaFeatures), decayedReplyCount = getDoubleAsIntOptFeature(DECAYED_REPLY_COUNT, doubleSearchResultSchemaFeatures), decayedQuoteCount = getDoubleAsIntOptFeature(DECAYED_QUOTE_COUNT, doubleSearchResultSchemaFeatures), fakeFavoriteCount = getDoubleAsIntOptFeature(FAKE_FAVORITE_COUNT, doubleSearchResultSchemaFeatures), fakeRetweetCount = getDoubleAsIntOptFeature(FAKE_RETWEET_COUNT, doubleSearchResultSchemaFeatures), fakeReplyCount = getDoubleAsIntOptFeature(FAKE_REPLY_COUNT, doubleSearchResultSchemaFeatures), fakeQuoteCount = getDoubleAsIntOptFeature(FAKE_QUOTE_COUNT, doubleSearchResultSchemaFeatures), // Scores. textScore = metadata.textScore.getOrElse(DefaultScore), earlybirdScore = metadata.score.getOrElse(DefaultScore), parusScore = metadata.parusScore.getOrElse(DefaultScore), userRep = metadata.userRep.getOrElse(DefaultScore), pBlockScore = metadata.extraMetadata.flatMap(_.pBlockScore), toxicityScore = metadata.extraMetadata.flatMap(_.toxicityScore), pSpammyTweetScore = metadata.extraMetadata.flatMap(_.pSpammyTweetScore), pReportedTweetScore = metadata.extraMetadata.flatMap(_.pReportedTweetScore), pSpammyTweetContent = metadata.extraMetadata.flatMap(_.spammyTweetContentScore), // Safety Signals labelAbusiveFlag = getBooleanOptFeature(LABEL_ABUSIVE_FLAG, booleanSearchResultSchemaFeatures), labelAbusiveHiRclFlag = getBooleanOptFeature(LABEL_ABUSIVE_HI_RCL_FLAG, booleanSearchResultSchemaFeatures), labelDupContentFlag = getBooleanOptFeature(LABEL_DUP_CONTENT_FLAG, booleanSearchResultSchemaFeatures), labelNsfwHiPrcFlag = getBooleanOptFeature(LABEL_NSFW_HI_PRC_FLAG, booleanSearchResultSchemaFeatures), labelNsfwHiRclFlag = getBooleanOptFeature(LABEL_NSFW_HI_RCL_FLAG, booleanSearchResultSchemaFeatures), labelSpamFlag = getBooleanOptFeature(LABEL_SPAM_FLAG, booleanSearchResultSchemaFeatures), labelSpamHiRclFlag = getBooleanOptFeature(LABEL_SPAM_HI_RCL_FLAG, booleanSearchResultSchemaFeatures), // Periscope Features periscopeExists = getBooleanOptFeature(PERISCOPE_EXISTS, booleanSearchResultSchemaFeatures), periscopeHasBeenFeatured = getBooleanOptFeature(PERISCOPE_HAS_BEEN_FEATURED, booleanSearchResultSchemaFeatures), periscopeIsCurrentlyFeatured = getBooleanOptFeature( PERISCOPE_IS_CURRENTLY_FEATURED, booleanSearchResultSchemaFeatures), periscopeIsFromQualitySource = getBooleanOptFeature( PERISCOPE_IS_FROM_QUALITY_SOURCE, booleanSearchResultSchemaFeatures), periscopeIsLive = getBooleanOptFeature(PERISCOPE_IS_LIVE, booleanSearchResultSchemaFeatures), // Last Engagement Features lastFavSinceCreationHrs = getIntOptFeature(LAST_FAVORITE_SINCE_CREATION_HRS, intSearchResultSchemaFeatures), lastRetweetSinceCreationHrs = getIntOptFeature(LAST_RETWEET_SINCE_CREATION_HRS, intSearchResultSchemaFeatures), lastReplySinceCreationHrs = getIntOptFeature(LAST_REPLY_SINCE_CREATION_HRS, intSearchResultSchemaFeatures), lastQuoteSinceCreationHrs = getIntOptFeature(LAST_QUOTE_SINCE_CREATION_HRS, intSearchResultSchemaFeatures), likedByUserIds = metadata.extraMetadata.flatMap(_.likedByUserIds), isComposerSourceCamera = getBooleanOptFeature(COMPOSER_SOURCE_IS_CAMERA_FLAG, booleanSearchResultSchemaFeatures), ) } .getOrElse(thriftTweetFeatures) features } private def applyUserDependentFeatures( searcherUserId: Long, screenName: Option[String], userLanguages: Seq[scc.ThriftLanguage], uiLanguage: Option[scc.ThriftLanguage], tweetCountByAuthorId: Map[Long, Int], followedUserIds: Set[Long], mutuallyFollowingUserIds: Set[Long], idToSearchResults: Map[Long, eb.ThriftSearchResult], inNetworkEngagement: InNetworkEngagement, result: eb.ThriftSearchResult )( thriftTweetFeatures: sc.ThriftTweetFeatures ): sc.ThriftTweetFeatures = { result.metadata .map { metadata => val isRetweet = metadata.isRetweet.getOrElse(false) val sourceTweet = if (isRetweet) idToSearchResults.get(metadata.sharedStatusId) else None val mentionsInSourceTweet = sourceTweet.map(getMentions).getOrElse(Seq.empty) val isReply = metadata.isReply.getOrElse(false) val replyToSearcher = isReply && (metadata.referencedTweetAuthorId == searcherUserId) val replyOther = isReply && !replyToSearcher val retweetOther = isRetweet && (metadata.referencedTweetAuthorId != searcherUserId) val tweetLanguage = metadata.language.getOrElse(scc.ThriftLanguage.Unknown) val referencedTweetAuthorId = if (metadata.referencedTweetAuthorId > 0) Some(metadata.referencedTweetAuthorId) else None val inReplyToUserId = if (!isRetweet) referencedTweetAuthorId else None thriftTweetFeatures.copy( // Info about the Tweet. fromSearcher = metadata.fromUserId == searcherUserId, probablyFromFollowedAuthor = followedUserIds.contains(metadata.fromUserId), fromMutualFollow = mutuallyFollowingUserIds.contains(metadata.fromUserId), replySearcher = replyToSearcher, replyOther = replyOther, retweetOther = retweetOther, mentionSearcher = isUserMentioned(screenName, getMentions(result), mentionsInSourceTweet), // Info about Tweet content/media. matchesSearcherMainLang = isUsersMainLanguage(tweetLanguage, userLanguages), matchesSearcherLangs = isUsersLanguage(tweetLanguage, userLanguages), matchesUILang = isUILanguage(tweetLanguage, uiLanguage), // Various counts. prevUserTweetEngagement = metadata.extraMetadata.flatMap(_.prevUserTweetEngagement).getOrElse(DefaultCount), tweetCountFromUserInSnapshot = tweetCountByAuthorId(metadata.fromUserId), bidirectionalReplyCount = inNetworkEngagement.biDirectionalReplyCounts(result.id), unidirectionalReplyCount = inNetworkEngagement.uniDirectionalReplyCounts(result.id), bidirectionalRetweetCount = inNetworkEngagement.biDirectionalRetweetCounts(result.id), unidirectionalRetweetCount = inNetworkEngagement.uniDirectionalRetweetCounts(result.id), conversationCount = inNetworkEngagement.descendantReplyCounts(result.id), directedAtUserIdIsInFirstDegree = if (isReply) inReplyToUserId.map(followedUserIds.contains) else None, ) } .getOrElse(thriftTweetFeatures) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird/RelevanceSearchUtil.scala ================================================ package com.twitter.home_mixer.util.earlybird import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant import com.twitter.search.earlybird.{thriftscala => eb} object RelevanceSearchUtil { val Mentions: String = EarlybirdFieldConstant.MENTIONS_FACET val FacetsToFetch: Seq[String] = Seq(Mentions) val MaxHitsToProcess: Int = 1500 val MetadataOptions: eb.ThriftSearchResultMetadataOptions = { eb.ThriftSearchResultMetadataOptions( getTweetUrls = true, getResultLocation = false, getLuceneScore = false, getInReplyToStatusId = true, getReferencedTweetAuthorId = true, getMediaBits = true, getAllFeatures = true, returnSearchResultFeatures = true, getExclusiveConversationAuthorId = true ) } val RelevanceOptions: eb.ThriftSearchRelevanceOptions = { eb.ThriftSearchRelevanceOptions( proximityScoring = true, maxConsecutiveSameUser = Some(2), rankingParams = None, maxHitsToProcess = Some(MaxHitsToProcess), maxUserBlendCount = Some(3), proximityPhraseWeight = 9.0, returnAllResults = Some(true) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/thrift/src/main/thrift:thrift-scala", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/RequestFields.scala ================================================ package com.twitter.home_mixer.util.tweetypie import com.twitter.tweetypie.{thriftscala => tp} object RequestFields { val CoreTweetFields: Set[tp.TweetInclude] = Set[tp.TweetInclude]( tp.TweetInclude.TweetFieldId(tp.Tweet.IdField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.CoreDataField.id) ) val MediaFields: Set[tp.TweetInclude] = Set[tp.TweetInclude]( tp.TweetInclude.TweetFieldId(tp.Tweet.MediaField.id), ) val SelfThreadFields: Set[tp.TweetInclude] = Set[tp.TweetInclude]( tp.TweetInclude.TweetFieldId(tp.Tweet.SelfThreadMetadataField.id) ) val MentionsTweetFields: Set[tp.TweetInclude] = Set[tp.TweetInclude]( tp.TweetInclude.TweetFieldId(tp.Tweet.MentionsField.id) ) val SemanticAnnotationTweetFields: Set[tp.TweetInclude] = Set[tp.TweetInclude]( tp.TweetInclude.TweetFieldId(tp.Tweet.EscherbirdEntityAnnotationsField.id) ) val NsfwLabelFields: Set[tp.TweetInclude] = Set[tp.TweetInclude]( // Tweet fields containing NSFW related attributes. tp.TweetInclude.TweetFieldId(tp.Tweet.NsfwHighRecallLabelField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.NsfwHighPrecisionLabelField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.NsfaHighRecallLabelField.id) ) val SafetyLabelFields: Set[tp.TweetInclude] = Set[tp.TweetInclude]( // Tweet fields containing RTF labels for abuse and spam. tp.TweetInclude.TweetFieldId(tp.Tweet.SpamLabelField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.AbusiveLabelField.id) ) val ConversationControlField: Set[tp.TweetInclude] = Set[tp.TweetInclude](tp.TweetInclude.TweetFieldId(tp.Tweet.ConversationControlField.id)) val TweetTPHydrationFields: Set[tp.TweetInclude] = CoreTweetFields ++ NsfwLabelFields ++ SafetyLabelFields ++ SemanticAnnotationTweetFields ++ Set( tp.TweetInclude.TweetFieldId(tp.Tweet.TakedownCountryCodesField.id), // QTs imply a TweetyPie -> SGS request dependency tp.TweetInclude.TweetFieldId(tp.Tweet.QuotedTweetField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.CommunitiesField.id), // Field required for determining if a Tweet was created via News Camera. tp.TweetInclude.TweetFieldId(tp.Tweet.ComposerSourceField.id), tp.TweetInclude.TweetFieldId(tp.Tweet.LanguageField.id) ) val TweetStaticEntitiesFields: Set[tp.TweetInclude] = MentionsTweetFields ++ CoreTweetFields ++ SemanticAnnotationTweetFields ++ MediaFields val ContentFields: Set[tp.TweetInclude] = CoreTweetFields ++ MediaFields ++ SelfThreadFields ++ ConversationControlField ++ SemanticAnnotationTweetFields ++ Set[tp.TweetInclude]( tp.TweetInclude.MediaEntityFieldId(tp.MediaEntity.AdditionalMetadataField.id)) } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/BUILD.bazel ================================================ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", ], ) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/FeatureExtractionHelper.scala ================================================ package com.twitter.home_mixer.util.tweetypie.content import com.twitter.home_mixer.model.ContentFeatures import com.twitter.tweetypie.{thriftscala => tp} object FeatureExtractionHelper { def extractFeatures( tweet: tp.Tweet, isExtractMediaEntities: Boolean = true ): ContentFeatures = { val contentFeaturesFromTweet = ContentFeatures.Empty.copy( selfThreadMetadata = tweet.selfThreadMetadata ) val contentFeaturesWithText = TweetTextFeaturesExtractor.addTextFeaturesFromTweet( contentFeaturesFromTweet, tweet ) val contentFeaturesWithMedia = TweetMediaFeaturesExtractor.addMediaFeaturesFromTweet( contentFeaturesWithText, tweet, isExtractMediaEntities ) contentFeaturesWithMedia.copy( conversationControl = tweet.conversationControl, semanticCoreAnnotations = tweet.escherbirdEntityAnnotations.map(_.entityAnnotations) ) } } ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/TweetMediaFeaturesExtractor.scala ================================================ package com.twitter.home_mixer.util.tweetypie.content import com.twitter.home_mixer.model.ContentFeatures import com.twitter.mediaservices.commons.mediainformation.{thriftscala => mi} import com.twitter.mediaservices.commons.tweetmedia.{thriftscala => tm} import com.twitter.mediaservices.commons.{thriftscala => ms} import com.twitter.tweetypie.{thriftscala => tp} import scala.collection.Map object TweetMediaFeaturesExtractor { private val ImageCategories = Set( ms.MediaCategory.TweetImage.value, ms.MediaCategory.TweetGif.value ) private val VideoCategories = Set( ms.MediaCategory.TweetVideo.value, ms.MediaCategory.AmplifyVideo.value ) def hasImage(tweet: tp.Tweet): Boolean = hasMediaByCategory(tweet, ImageCategories) def hasVideo(tweet: tp.Tweet): Boolean = hasMediaByCategory(tweet, VideoCategories) private def hasMediaByCategory(tweet: tp.Tweet, categories: Set[Int]): Boolean = { tweet.media.exists { mediaEntities => mediaEntities.exists { mediaEntity => mediaEntity.mediaKey.map(_.mediaCategory).exists { mediaCategory => categories.contains(mediaCategory.value) } } } } def getMediaIds(tweet: tp.Tweet): Seq[Long] = { val tweetMediaIdsOpt = tweet.media.map { mediaEntities => mediaEntities.map { mediaEntity => mediaEntity.mediaId } } tweetMediaIdsOpt.getOrElse(Seq.empty) } def addMediaFeaturesFromTweet( inputFeatures: ContentFeatures, tweet: tp.Tweet, isExtractMediaEntities: Boolean = true ): ContentFeatures = { val featuresWithMediaEntity = if (isExtractMediaEntities) { tweet.media .map { mediaEntities => val sizeFeatures = getSizeFeatures(mediaEntities) val playbackFeatures = getPlaybackFeatures(mediaEntities) val mediaWidths = sizeFeatures.map(_.width.toShort) val mediaHeights = sizeFeatures.map(_.height.toShort) val resizeMethods = sizeFeatures.map(_.resizeMethod.toShort) val faceMapAreas = getFaceMapAreas(mediaEntities) val sortedColorPalette = getSortedColorPalette(mediaEntities) val stickerFeatures = getStickerFeatures(mediaEntities) val mediaOriginProviders = getMediaOriginProviders(mediaEntities) val isManaged = getIsManaged(mediaEntities) val is360 = getIs360(mediaEntities) val viewCount = getViewCount(mediaEntities) val userDefinedProductMetadataFeatures = getUserDefinedProductMetadataFeatures(mediaEntities) val isMonetizable = getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.isMonetizable)) val isEmbeddable = getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.isEmbeddable)) val hasSelectedPreviewImage = getOptBooleanFromSeqOpt( userDefinedProductMetadataFeatures.map(_.hasSelectedPreviewImage)) val hasImage = Some( mediaEntities.exists { entity => entity.mediaKey.exists { key => ImageCategories.contains(key.mediaCategory.value) } } ) val hasVideo = Some( mediaEntities.exists { entity => entity.mediaKey.exists { key => VideoCategories.contains(key.mediaCategory.value) } } ) val hasTitle = getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.hasTitle)) val hasDescription = getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.hasDescription)) val hasVisitSiteCallToAction = getOptBooleanFromSeqOpt( userDefinedProductMetadataFeatures.map(_.hasVisitSiteCallToAction)) val hasAppInstallCallToAction = getOptBooleanFromSeqOpt( userDefinedProductMetadataFeatures.map(_.hasAppInstallCallToAction)) val hasWatchNowCallToAction = getOptBooleanFromSeqOpt( userDefinedProductMetadataFeatures.map(_.hasWatchNowCallToAction)) inputFeatures.copy( videoDurationMs = playbackFeatures.durationMs, bitRate = playbackFeatures.bitRate, aspectRatioNum = playbackFeatures.aspectRatioNum, aspectRatioDen = playbackFeatures.aspectRatioDen, widths = Some(mediaWidths), heights = Some(mediaHeights), resizeMethods = Some(resizeMethods), faceAreas = Some(faceMapAreas), dominantColorRed = sortedColorPalette.headOption.map(_.rgb.red), dominantColorBlue = sortedColorPalette.headOption.map(_.rgb.blue), dominantColorGreen = sortedColorPalette.headOption.map(_.rgb.green), dominantColorPercentage = sortedColorPalette.headOption.map(_.percentage), numColors = Some(sortedColorPalette.size.toShort), stickerIds = Some(stickerFeatures), mediaOriginProviders = Some(mediaOriginProviders), isManaged = Some(isManaged), is360 = Some(is360), viewCount = viewCount, isMonetizable = isMonetizable, isEmbeddable = isEmbeddable, hasSelectedPreviewImage = hasSelectedPreviewImage, hasTitle = hasTitle, hasDescription = hasDescription, hasVisitSiteCallToAction = hasVisitSiteCallToAction, hasAppInstallCallToAction = hasAppInstallCallToAction, hasWatchNowCallToAction = hasWatchNowCallToAction, hasImage = hasImage, hasVideo = hasVideo ) } .getOrElse(inputFeatures) } else inputFeatures val featuresWithMediaTags = tweet.mediaTags .map { mediaTags => val mediaTagScreenNames = getMediaTagScreenNames(mediaTags.tagMap) val numMediaTags = mediaTagScreenNames.size featuresWithMediaEntity.copy( mediaTagScreenNames = Some(mediaTagScreenNames), numMediaTags = Some(numMediaTags.toShort) ) } .getOrElse(featuresWithMediaEntity) featuresWithMediaTags .copy(media = tweet.media) } private def getSizeFeatures(mediaEntities: Seq[tp.MediaEntity]): Seq[MediaSizeFeatures] = { mediaEntities.map { mediaEntity => mediaEntity.sizes.foldLeft(MediaSizeFeatures(0, 0, 0))((accDimensions, dimensions) => MediaSizeFeatures( width = math.max(dimensions.width, accDimensions.width), height = math.max(dimensions.height, accDimensions.height), resizeMethod = math.max(dimensions.resizeMethod.getValue, accDimensions.resizeMethod) )) } } private def getPlaybackFeatures(mediaEntities: Seq[tp.MediaEntity]): PlaybackFeatures = { val allPlaybackFeatures = mediaEntities .flatMap { mediaEntity => mediaEntity.mediaInfo map { case videoEntity: tm.MediaInfo.VideoInfo => PlaybackFeatures( durationMs = Some(videoEntity.videoInfo.durationMillis), bitRate = videoEntity.videoInfo.variants.maxBy(_.bitRate).bitRate, aspectRatioNum = Some(videoEntity.videoInfo.aspectRatio.numerator), aspectRatioDen = Some(videoEntity.videoInfo.aspectRatio.denominator) ) case gifEntity: tm.MediaInfo.AnimatedGifInfo => PlaybackFeatures( durationMs = None, bitRate = gifEntity.animatedGifInfo.variants.maxBy(_.bitRate).bitRate, aspectRatioNum = Some(gifEntity.animatedGifInfo.aspectRatio.numerator), aspectRatioDen = Some(gifEntity.animatedGifInfo.aspectRatio.denominator) ) case _ => PlaybackFeatures(None, None, None, None) } } .collect { case playbackFeatures: PlaybackFeatures => playbackFeatures } if (allPlaybackFeatures.nonEmpty) allPlaybackFeatures.minBy(_.durationMs) else PlaybackFeatures(None, None, None, None) } private def getMediaTagScreenNames(tagMap: Map[Long, Seq[tp.MediaTag]]): Seq[String] = tagMap.values .flatMap(seqMediaTag => seqMediaTag.flatMap(_.screenName)) .toSeq // Areas of the faces identified in the media entities private def getFaceMapAreas(mediaEntities: Seq[tp.MediaEntity]): Seq[Int] = { for { mediaEntity <- mediaEntities metadata <- mediaEntity.additionalMetadata.toSeq faceData <- metadata.faceData faces <- faceData.faces } yield { faces .getOrElse("orig", Seq.empty[mi.Face]) .flatMap(f => f.boundingBox.map(bb => bb.width * bb.height)) } }.flatten // All ColorPalettes in the media sorted by the percentage in descending order private def getSortedColorPalette( mediaEntities: Seq[tp.MediaEntity] ): Seq[mi.ColorPaletteItem] = { for { mediaEntity <- mediaEntities metadata <- mediaEntity.additionalMetadata.toSeq colorInfo <- metadata.colorInfo } yield { colorInfo.palette } }.flatten.sortBy(-_.percentage) // Id's of stickers applied by the user private def getStickerFeatures(mediaEntities: Seq[tp.MediaEntity]): Seq[Long] = { for { mediaEntity <- mediaEntities metadata <- mediaEntity.additionalMetadata.toSeq stickerInfo <- metadata.stickerInfo } yield { stickerInfo.stickers.map(_.id) } }.flatten // 3rd party media providers. eg. giphy for gifs private def getMediaOriginProviders(mediaEntities: Seq[tp.MediaEntity]): Seq[String] = for { mediaEntity <- mediaEntities metadata <- mediaEntity.additionalMetadata.toSeq mediaOrigin <- metadata.foundMediaOrigin } yield { mediaOrigin.provider } private def getIsManaged(mediaEntities: Seq[tp.MediaEntity]): Boolean = { for { mediaEntity <- mediaEntities metadata <- mediaEntity.additionalMetadata.toSeq managementInfo <- metadata.managementInfo } yield { managementInfo.managed } }.contains(true) private def getIs360(mediaEntities: Seq[tp.MediaEntity]): Boolean = { for { mediaEntity <- mediaEntities metadata <- mediaEntity.additionalMetadata.toSeq info360 <- metadata.info360 } yield { info360.is360 } }.contains(Some(true)) private def getViewCount(mediaEntities: Seq[tp.MediaEntity]): Option[Long] = { for { mediaEntity <- mediaEntities metadata <- mediaEntity.additionalMetadata.toSeq engagementInfo <- metadata.engagementInfo viewCounts <- engagementInfo.viewCount } yield { viewCounts } }.reduceOption(_ max _) // metadata defined by the user when uploading the image private def getUserDefinedProductMetadataFeatures( mediaEntities: Seq[tp.MediaEntity] ): Seq[UserDefinedProductMetadataFeatures] = for { mediaEntity <- mediaEntities userDefinedMetadata <- mediaEntity.metadata } yield { UserDefinedProductMetadataFeatures( isMonetizable = userDefinedMetadata.monetizable, isEmbeddable = userDefinedMetadata.embeddable, hasSelectedPreviewImage = Some(userDefinedMetadata.previewImage.nonEmpty), hasTitle = userDefinedMetadata.title.map(_.nonEmpty), hasDescription = userDefinedMetadata.description.map(_.nonEmpty), hasVisitSiteCallToAction = userDefinedMetadata.callToActions.map(_.visitSite.nonEmpty), hasAppInstallCallToAction = userDefinedMetadata.callToActions.map(_.appInstall.nonEmpty), hasWatchNowCallToAction = userDefinedMetadata.callToActions.map(_.watchNow.nonEmpty) ) } private def getOptBooleanFromSeqOpt( seqOpt: Seq[Option[Boolean]] ): Option[Boolean] = Some( seqOpt.exists(boolOpt => boolOpt.contains(true)) ) } case class MediaSizeFeatures(width: Int, height: Int, resizeMethod: Int) case class PlaybackFeatures( durationMs: Option[Int], bitRate: Option[Int], aspectRatioNum: Option[Short], aspectRatioDen: Option[Short]) case class UserDefinedProductMetadataFeatures( isMonetizable: Option[Boolean], isEmbeddable: Option[Boolean], hasSelectedPreviewImage: Option[Boolean], hasTitle: Option[Boolean], hasDescription: Option[Boolean], hasVisitSiteCallToAction: Option[Boolean], hasAppInstallCallToAction: Option[Boolean], hasWatchNowCallToAction: Option[Boolean]) ================================================ FILE: home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content/TweetTextFeaturesExtractor.scala ================================================ package com.twitter.home_mixer.util.tweetypie.content import com.twitter.home_mixer.model.ContentFeatures import com.twitter.tweetypie.{thriftscala => tp} object TweetTextFeaturesExtractor { private val QUESTION_MARK_CHARS = Set( '\u003F', '\u00BF', '\u037E', '\u055E', '\u061F', '\u1367', '\u1945', '\u2047', '\u2048', '\u2049', '\u2753', '\u2754', '\u2CFA', '\u2CFB', '\u2E2E', '\uA60F', '\uA6F7', '\uFE16', '\uFE56', '\uFF1F', '\u1114', '\u1E95' ) private val NEW_LINE_REGEX = "\r\n|\r|\n".r def addTextFeaturesFromTweet( inputFeatures: ContentFeatures, tweet: tp.Tweet ): ContentFeatures = { tweet.coreData .map { coreData => val tweetText = coreData.text inputFeatures.copy( hasQuestion = hasQuestionCharacter(tweetText), length = getLength(tweetText).toShort, numCaps = getCaps(tweetText).toShort, numWhiteSpaces = getSpaces(tweetText).toShort, numNewlines = Some(getNumNewlines(tweetText)), ) } .getOrElse(inputFeatures) } def getLength(text: String): Int = text.codePointCount(0, text.length()) def getCaps(text: String): Int = text.count(Character.isUpperCase) def getSpaces(text: String): Int = text.count(Character.isWhitespace) def hasQuestionCharacter(text: String): Boolean = text.exists(QUESTION_MARK_CHARS.contains) def getNumNewlines(text: String): Short = NEW_LINE_REGEX.findAllIn(text).length.toShort } ================================================ FILE: navi/README.md ================================================ # Navi: High-Performance Machine Learning Serving Server in Rust Navi is a high-performance, versatile machine learning serving server implemented in Rust and tailored for production usage. It's designed to efficiently serve within the Twitter tech stack, offering top-notch performance while focusing on core features. ## Key Features - **Minimalist Design Optimized for Production Use Cases**: Navi delivers ultra-high performance, stability, and availability, engineered to handle real-world application demands with a streamlined codebase. - **gRPC API Compatibility with TensorFlow Serving**: Seamless integration with existing TensorFlow Serving clients via its gRPC API, enabling easy integration, smooth deployment, and scaling in production environments. - **Plugin Architecture for Different Runtimes**: Navi's pluggable architecture supports various machine learning runtimes, providing adaptability and extensibility for diverse use cases. Out-of-the-box support is available for TensorFlow and Onnx Runtime, with PyTorch in an experimental state. ## Current State While Navi's features may not be as comprehensive as its open-source counterparts, its performance-first mindset makes it highly efficient. - Navi for TensorFlow is currently the most feature-complete, supporting multiple input tensors of different types (float, int, string, etc.). - Navi for Onnx primarily supports one input tensor of type string, used in Twitter's home recommendation with a proprietary BatchPredictRequest format. - Navi for Pytorch is compilable and runnable but not yet production-ready in terms of performance and stability. ## Directory Structure - `navi`: The main code repository for Navi - `dr_transform`: Twitter-specific converter that converts BatchPredictionRequest Thrift to ndarray - `segdense`: Twitter-specific config to specify how to retrieve feature values from BatchPredictionRequest - `thrift_bpr_adapter`: generated thrift code for BatchPredictionRequest ## Content We have included all *.rs source code files that make up the main Navi binaries for you to examine. However, we have not included the test and benchmark code, or various configuration files, due to data security concerns. ## Run In navi/navi, you can run the following commands: - `scripts/run_tf2.sh` for [TensorFlow](https://www.tensorflow.org/) - `scripts/run_onnx.sh` for [Onnx](https://onnx.ai/) Do note that you need to create a models directory and create some versions, preferably using epoch time, e.g., `1679693908377`. so the models structure looks like: models/ -web_click - 1809000 - 1809010 ## Build You can adapt the above scripts to build using Cargo. ================================================ FILE: navi/dr_transform/Cargo.toml ================================================ [package] name = "dr_transform" version = "0.1.0" edition = "2021" [dependencies] serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" json = "0.12.4" bpr_thrift = { path = "../thrift_bpr_adapter/thrift/"} segdense = { path = "../segdense/"} thrift = "0.17.0" ndarray = "0.15" base64 = "0.20.0" npyz = "0.7.2" log = "0.4.17" env_logger = "0.9.0" prometheus = "0.13.1" once_cell = "1.17.0" rand = "0.8.5" itertools = "0.10.5" anyhow = "1.0.70" [target.'cfg(not(target_os="linux"))'.dependencies] ort = {git ="https://github.com/pykeio/ort.git", features=["profiling"], tag="v1.14.6"} [target.'cfg(target_os="linux")'.dependencies] ort = {git ="https://github.com/pykeio/ort.git", features=["profiling", "tensorrt", "cuda", "copy-dylibs"], tag="v1.14.6"} [dev-dependencies] criterion = "0.3.0" [[bench]] name = "bpr_benchmark" harness = false ================================================ FILE: navi/dr_transform/src/all_config.rs ================================================ use serde::{Deserialize, Serialize}; use serde_json::Error; #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct AllConfig { #[serde(rename = "train_data")] pub train_data: TrainData, } #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TrainData { #[serde(rename = "seg_dense_schema")] pub seg_dense_schema: SegDenseSchema, } #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SegDenseSchema { #[serde(rename = "renamed_features")] pub renamed_features: RenamedFeatures, } #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct RenamedFeatures { pub continuous: String, pub binary: String, pub discrete: String, #[serde(rename = "author_embedding")] pub author_embedding: String, #[serde(rename = "user_embedding")] pub user_embedding: String, #[serde(rename = "user_eng_embedding")] pub user_eng_embedding: String, #[serde(rename = "meta__author_id")] pub meta_author_id: String, #[serde(rename = "meta__user_id")] pub meta_user_id: String, #[serde(rename = "meta__tweet_id")] pub meta_tweet_id: String, } pub fn parse(json_str: &str) -> Result { let all_config: AllConfig = serde_json::from_str(json_str)?; Ok(all_config) } ================================================ FILE: navi/dr_transform/src/converter.rs ================================================ use std::collections::BTreeSet; use std::fmt::{self, Debug, Display}; use std::fs; use crate::all_config; use crate::all_config::AllConfig; use anyhow::{bail, Context}; use bpr_thrift::data::DataRecord; use bpr_thrift::prediction_service::BatchPredictionRequest; use bpr_thrift::tensor::GeneralTensor; use log::debug; use ndarray::Array2; use once_cell::sync::OnceCell; use ort::tensor::InputTensor; use prometheus::{HistogramOpts, HistogramVec}; use segdense::mapper::{FeatureMapper, MapReader}; use segdense::segdense_transform_spec_home_recap_2022::{DensificationTransformSpec, Root}; use segdense::util; use thrift::protocol::{TBinaryInputProtocol, TSerializable}; use thrift::transport::TBufferChannel; pub fn log_feature_match( dr: &DataRecord, seg_dense_config: &DensificationTransformSpec, dr_type: String, ) { // Note the following algorithm matches features from config using linear search. // Also the record source is MinDataRecord. This includes only binary and continous features for now. for (feature_id, feature_value) in dr.continuous_features.as_ref().unwrap() { debug!( "{} - Continous Datarecord => Feature ID: {}, Feature value: {}", dr_type, feature_id, feature_value ); for input_feature in &seg_dense_config.cont.input_features { if input_feature.feature_id == *feature_id { debug!("Matching input feature: {:?}", input_feature) } } } for feature_id in dr.binary_features.as_ref().unwrap() { debug!( "{} - Binary Datarecord => Feature ID: {}", dr_type, feature_id ); for input_feature in &seg_dense_config.binary.input_features { if input_feature.feature_id == *feature_id { debug!("Found input feature: {:?}", input_feature) } } } } pub fn log_feature_matches(drs: &Vec, seg_dense_config: &DensificationTransformSpec) { for dr in drs { log_feature_match(dr, seg_dense_config, String::from("individual")); } } pub trait Converter: Send + Sync + Debug + 'static + Display { fn convert(&self, input: Vec>) -> (Vec, Vec); } #[derive(Debug)] #[allow(dead_code)] pub struct BatchPredictionRequestToTorchTensorConverter { all_config: AllConfig, seg_dense_config: Root, all_config_path: String, seg_dense_config_path: String, feature_mapper: FeatureMapper, user_embedding_feature_id: i64, user_eng_embedding_feature_id: i64, author_embedding_feature_id: i64, discrete_features_to_report: BTreeSet, continuous_features_to_report: BTreeSet, discrete_feature_metrics: &'static HistogramVec, continuous_feature_metrics: &'static HistogramVec, } impl Display for BatchPredictionRequestToTorchTensorConverter { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!( f, "all_config_path: {}, seg_dense_config_path:{}", self.all_config_path, self.seg_dense_config_path ) } } impl BatchPredictionRequestToTorchTensorConverter { pub fn new( model_dir: &str, model_version: &str, reporting_feature_ids: Vec<(i64, &str)>, register_metric_fn: Option, ) -> anyhow::Result { let all_config_path = format!("{}/{}/all_config.json", model_dir, model_version); let seg_dense_config_path = format!( "{}/{}/segdense_transform_spec_home_recap_2022.json", model_dir, model_version ); let seg_dense_config = util::load_config(&seg_dense_config_path)?; let all_config = all_config::parse( &fs::read_to_string(&all_config_path) .with_context(|| "error loading all_config.json - ")?, )?; let feature_mapper = util::load_from_parsed_config(seg_dense_config.clone())?; let user_embedding_feature_id = Self::get_feature_id( &all_config .train_data .seg_dense_schema .renamed_features .user_embedding, &seg_dense_config, ); let user_eng_embedding_feature_id = Self::get_feature_id( &all_config .train_data .seg_dense_schema .renamed_features .user_eng_embedding, &seg_dense_config, ); let author_embedding_feature_id = Self::get_feature_id( &all_config .train_data .seg_dense_schema .renamed_features .author_embedding, &seg_dense_config, ); static METRICS: OnceCell<(HistogramVec, HistogramVec)> = OnceCell::new(); let (discrete_feature_metrics, continuous_feature_metrics) = METRICS.get_or_init(|| { let discrete = HistogramVec::new( HistogramOpts::new(":navi:feature_id:discrete", "Discrete Feature ID values") .buckets(Vec::from(&[ 0.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0, 110.0, 120.0, 130.0, 140.0, 150.0, 160.0, 170.0, 180.0, 190.0, 200.0, 250.0, 300.0, 500.0, 1000.0, 10000.0, 100000.0, ] as &'static [f64])), &["feature_id"], ) .expect("metric cannot be created"); let continuous = HistogramVec::new( HistogramOpts::new( ":navi:feature_id:continuous", "continuous Feature ID values", ) .buckets(Vec::from(&[ 0.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0, 110.0, 120.0, 130.0, 140.0, 150.0, 160.0, 170.0, 180.0, 190.0, 200.0, 250.0, 300.0, 500.0, 1000.0, 10000.0, 100000.0, ] as &'static [f64])), &["feature_id"], ) .expect("metric cannot be created"); register_metric_fn.map(|r| { r(&discrete); r(&continuous); }); (discrete, continuous) }); let mut discrete_features_to_report = BTreeSet::new(); let mut continuous_features_to_report = BTreeSet::new(); for (feature_id, feature_type) in reporting_feature_ids.iter() { match *feature_type { "discrete" => discrete_features_to_report.insert(feature_id.clone()), "continuous" => continuous_features_to_report.insert(feature_id.clone()), _ => bail!( "Invalid feature type {} for reporting metrics!", feature_type ), }; } Ok(BatchPredictionRequestToTorchTensorConverter { all_config, seg_dense_config, all_config_path, seg_dense_config_path, feature_mapper, user_embedding_feature_id, user_eng_embedding_feature_id, author_embedding_feature_id, discrete_features_to_report, continuous_features_to_report, discrete_feature_metrics, continuous_feature_metrics, }) } fn get_feature_id(feature_name: &str, seg_dense_config: &Root) -> i64 { // given a feature name, we get the complex feature type id for feature in &seg_dense_config.complex_feature_type_transform_spec { if feature.full_feature_name == feature_name { return feature.feature_id; } } -1 } fn parse_batch_prediction_request(bytes: Vec) -> BatchPredictionRequest { // parse batch prediction request into a struct from byte array repr. let mut bc = TBufferChannel::with_capacity(bytes.len(), 0); bc.set_readable_bytes(&bytes); let mut protocol = TBinaryInputProtocol::new(bc, true); BatchPredictionRequest::read_from_in_protocol(&mut protocol).unwrap() } fn get_embedding_tensors( &self, bprs: &[BatchPredictionRequest], feature_id: i64, batch_size: &[usize], ) -> Array2 { // given an embedding feature id, extract the float tensor array into tensors. let cols: usize = 200; let rows: usize = batch_size[batch_size.len() - 1]; let total_size = rows * cols; let mut working_set = vec![0 as f32; total_size]; let mut bpr_start = 0; for (bpr, &bpr_end) in bprs.iter().zip(batch_size) { if bpr.common_features.is_some() { if bpr.common_features.as_ref().unwrap().tensors.is_some() { if bpr .common_features .as_ref() .unwrap() .tensors .as_ref() .unwrap() .contains_key(&feature_id) { let source_tensor = bpr .common_features .as_ref() .unwrap() .tensors .as_ref() .unwrap() .get(&feature_id) .unwrap(); let tensor = match source_tensor { GeneralTensor::FloatTensor(float_tensor) => //Tensor::of_slice( { float_tensor .floats .iter() .map(|x| x.into_inner() as f32) .collect::>() } _ => vec![0 as f32; cols], }; // since the tensor is found in common feature, add it in all batches for row in bpr_start..bpr_end { for col in 0..cols { working_set[row * cols + col] = tensor[col]; } } } } } // find the feature in individual feature list and add to corresponding batch. for (index, datarecord) in bpr.individual_features_list.iter().enumerate() { if datarecord.tensors.is_some() && datarecord .tensors .as_ref() .unwrap() .contains_key(&feature_id) { let source_tensor = datarecord .tensors .as_ref() .unwrap() .get(&feature_id) .unwrap(); let tensor = match source_tensor { GeneralTensor::FloatTensor(float_tensor) => float_tensor .floats .iter() .map(|x| x.into_inner() as f32) .collect::>(), _ => vec![0 as f32; cols], }; for col in 0..cols { working_set[(bpr_start + index) * cols + col] = tensor[col]; } } } bpr_start = bpr_end; } Array2::::from_shape_vec([rows, cols], working_set).unwrap() } // Todo : Refactor, create a generic version with different type and field accessors // Example paramterize and then instiantiate the following // (FLOAT --> FLOAT, DataRecord.continuous_feature) // (BOOL --> INT64, DataRecord.binary_feature) // (INT64 --> INT64, DataRecord.discrete_feature) fn get_continuous(&self, bprs: &[BatchPredictionRequest], batch_ends: &[usize]) -> InputTensor { // These need to be part of model schema let rows: usize = batch_ends[batch_ends.len() - 1]; let cols: usize = 5293; let full_size: usize = rows * cols; let default_val = f32::NAN; let mut tensor = vec![default_val; full_size]; let mut bpr_start = 0; for (bpr, &bpr_end) in bprs.iter().zip(batch_ends) { // Common features if bpr.common_features.is_some() && bpr .common_features .as_ref() .unwrap() .continuous_features .is_some() { let common_features = bpr .common_features .as_ref() .unwrap() .continuous_features .as_ref() .unwrap(); for feature in common_features { match self.feature_mapper.get(feature.0) { Some(f_info) => { let idx = f_info.index_within_tensor as usize; if idx < cols { // Set value in each row for r in bpr_start..bpr_end { let flat_index: usize = r * cols + idx; tensor[flat_index] = feature.1.into_inner() as f32; } } } None => (), } if self.continuous_features_to_report.contains(feature.0) { self.continuous_feature_metrics .with_label_values(&[feature.0.to_string().as_str()]) .observe(feature.1.into_inner()) } else if self.discrete_features_to_report.contains(feature.0) { self.discrete_feature_metrics .with_label_values(&[feature.0.to_string().as_str()]) .observe(feature.1.into_inner()) } } } // Process the batch of datarecords for r in bpr_start..bpr_end { let dr: &DataRecord = &bpr.individual_features_list[usize::try_from(r - bpr_start).unwrap()]; if dr.continuous_features.is_some() { for feature in dr.continuous_features.as_ref().unwrap() { match self.feature_mapper.get(&feature.0) { Some(f_info) => { let idx = f_info.index_within_tensor as usize; let flat_index: usize = r * cols + idx; if flat_index < tensor.len() && idx < cols { tensor[flat_index] = feature.1.into_inner() as f32; } } None => (), } if self.continuous_features_to_report.contains(feature.0) { self.continuous_feature_metrics .with_label_values(&[feature.0.to_string().as_str()]) .observe(feature.1.into_inner() as f64) } else if self.discrete_features_to_report.contains(feature.0) { self.discrete_feature_metrics .with_label_values(&[feature.0.to_string().as_str()]) .observe(feature.1.into_inner() as f64) } } } } bpr_start = bpr_end; } InputTensor::FloatTensor( Array2::::from_shape_vec([rows, cols], tensor) .unwrap() .into_dyn(), ) } fn get_binary(&self, bprs: &[BatchPredictionRequest], batch_ends: &[usize]) -> InputTensor { // These need to be part of model schema let rows: usize = batch_ends[batch_ends.len() - 1]; let cols: usize = 149; let full_size: usize = rows * cols; let default_val: i64 = 0; let mut v = vec![default_val; full_size]; let mut bpr_start = 0; for (bpr, &bpr_end) in bprs.iter().zip(batch_ends) { // Common features if bpr.common_features.is_some() && bpr .common_features .as_ref() .unwrap() .binary_features .is_some() { let common_features = bpr .common_features .as_ref() .unwrap() .binary_features .as_ref() .unwrap(); for feature in common_features { match self.feature_mapper.get(feature) { Some(f_info) => { let idx = f_info.index_within_tensor as usize; if idx < cols { // Set value in each row for r in bpr_start..bpr_end { let flat_index: usize = r * cols + idx; v[flat_index] = 1; } } } None => (), } } } // Process the batch of datarecords for r in bpr_start..bpr_end { let dr: &DataRecord = &bpr.individual_features_list[r - bpr_start]; if dr.binary_features.is_some() { for feature in dr.binary_features.as_ref().unwrap() { match self.feature_mapper.get(&feature) { Some(f_info) => { let idx = f_info.index_within_tensor as usize; let flat_index: usize = r * cols + idx; v[flat_index] = 1; } None => (), } } } } bpr_start = bpr_end; } InputTensor::Int64Tensor( Array2::::from_shape_vec([rows, cols], v) .unwrap() .into_dyn(), ) } #[allow(dead_code)] fn get_discrete(&self, bprs: &[BatchPredictionRequest], batch_ends: &[usize]) -> InputTensor { // These need to be part of model schema let rows: usize = batch_ends[batch_ends.len() - 1]; let cols: usize = 320; let full_size: usize = rows * cols; let default_val: i64 = 0; let mut v = vec![default_val; full_size]; let mut bpr_start = 0; for (bpr, &bpr_end) in bprs.iter().zip(batch_ends) { // Common features if bpr.common_features.is_some() && bpr .common_features .as_ref() .unwrap() .discrete_features .is_some() { let common_features = bpr .common_features .as_ref() .unwrap() .discrete_features .as_ref() .unwrap(); for feature in common_features { match self.feature_mapper.get(feature.0) { Some(f_info) => { let idx = f_info.index_within_tensor as usize; if idx < cols { // Set value in each row for r in bpr_start..bpr_end { let flat_index: usize = r * cols + idx; v[flat_index] = *feature.1; } } } None => (), } if self.discrete_features_to_report.contains(feature.0) { self.discrete_feature_metrics .with_label_values(&[feature.0.to_string().as_str()]) .observe(*feature.1 as f64) } } } // Process the batch of datarecords for r in bpr_start..bpr_end { let dr: &DataRecord = &bpr.individual_features_list[usize::try_from(r).unwrap()]; if dr.discrete_features.is_some() { for feature in dr.discrete_features.as_ref().unwrap() { match self.feature_mapper.get(&feature.0) { Some(f_info) => { let idx = f_info.index_within_tensor as usize; let flat_index: usize = r * cols + idx; if flat_index < v.len() && idx < cols { v[flat_index] = *feature.1; } } None => (), } if self.discrete_features_to_report.contains(feature.0) { self.discrete_feature_metrics .with_label_values(&[feature.0.to_string().as_str()]) .observe(*feature.1 as f64) } } } } bpr_start = bpr_end; } InputTensor::Int64Tensor( Array2::::from_shape_vec([rows, cols], v) .unwrap() .into_dyn(), ) } fn get_user_embedding( &self, bprs: &[BatchPredictionRequest], batch_ends: &[usize], ) -> InputTensor { InputTensor::FloatTensor( self.get_embedding_tensors(bprs, self.user_embedding_feature_id, batch_ends) .into_dyn(), ) } fn get_eng_embedding( &self, bpr: &[BatchPredictionRequest], batch_ends: &[usize], ) -> InputTensor { InputTensor::FloatTensor( self.get_embedding_tensors(bpr, self.user_eng_embedding_feature_id, batch_ends) .into_dyn(), ) } fn get_author_embedding( &self, bpr: &[BatchPredictionRequest], batch_ends: &[usize], ) -> InputTensor { InputTensor::FloatTensor( self.get_embedding_tensors(bpr, self.author_embedding_feature_id, batch_ends) .into_dyn(), ) } } impl Converter for BatchPredictionRequestToTorchTensorConverter { fn convert(&self, batched_bytes: Vec>) -> (Vec, Vec) { let bprs = batched_bytes .into_iter() .map(|bytes| { BatchPredictionRequestToTorchTensorConverter::parse_batch_prediction_request(bytes) }) .collect::>(); let batch_ends = bprs .iter() .map(|bpr| bpr.individual_features_list.len()) .scan(0usize, |acc, e| { //running total *acc = *acc + e; Some(*acc) }) .collect::>(); let t1 = self.get_continuous(&bprs, &batch_ends); let t2 = self.get_binary(&bprs, &batch_ends); //let _t3 = self.get_discrete(&bprs, &batch_ends); let t4 = self.get_user_embedding(&bprs, &batch_ends); let t5 = self.get_eng_embedding(&bprs, &batch_ends); let t6 = self.get_author_embedding(&bprs, &batch_ends); (vec![t1, t2, t4, t5, t6], batch_ends) } } ================================================ FILE: navi/dr_transform/src/lib.rs ================================================ pub mod all_config; pub mod converter; #[cfg(test)] mod test; pub mod util; pub extern crate ort; ================================================ FILE: navi/dr_transform/src/util.rs ================================================ use npyz::WriterBuilder; use npyz::{AutoSerialize, WriteOptions}; use std::io::BufWriter; use std::{ fs::File, io::{self, BufRead}, }; pub fn load_batch_prediction_request_base64(file_name: &str) -> Vec> { let file = File::open(file_name).expect("could not read file"); let mut result = vec![]; for (mut line_count, line) in io::BufReader::new(file).lines().enumerate() { line_count += 1; match base64::decode(line.unwrap().trim()) { Ok(payload) => result.push(payload), Err(err) => println!("error decoding line {file_name}:{line_count} - {err}"), } } println!("result len: {}", result.len()); result } pub fn save_to_npy(data: &[T], save_to: String) { let mut writer = WriteOptions::new() .default_dtype() .shape(&[data.len() as u64, 1]) .writer(BufWriter::new(File::create(save_to).unwrap())) .begin_nd() .unwrap(); writer.extend(data.to_owned()).unwrap(); writer.finish().unwrap(); } ================================================ FILE: navi/navi/Cargo.toml ================================================ [package] name = "navi" version = "2.0.45" edition = "2021" [[bin]] name = "navi" path = "src/bin/navi.rs" required-features=["tf"] [[bin]] name = "navi_torch" path = "src/bin/navi_torch.rs" required-features=["torch"] [[bin]] name = "navi_onnx" path = "src/bin/navi_onnx.rs" required-features=["onnx"] [[bin]] name = "navi_onnx_test" path = "src/bin/bin_tests/navi_onnx_test.rs" [[bin]] name = "navi_torch_test" path = "src/bin/bin_tests/navi_torch_test.rs" required-features=["torch"] [features] default=[] navi_console=[] torch=["tch"] onnx=[] tf=["tensorflow"] [dependencies] itertools = "0.10.5" anyhow = "1.0.57" arrayvec = "0.7.2" clap = { version = "4.0.32", features = ["derive"] } console-subscriber = "0.1.6" time = { version = "0.3.20", features = ["parsing"] } env_logger = "0.10.0" flamegraph = "0.6.1" fnv = "1.0.7" futures = { version = "0.3", default-features = false } image = "0.24.5" indexmap = "1.8.1" lazy_static = "1.4" libloading = "0.7" log = "0.4.17" ndarray-rand = "0.14.0" prometheus = "0.13.1" prost = "0.9" prost-types = "0.9" parking_lot = "0.12.1" rand = "0.8.5" rand_pcg = "0.3.1" random = "0.12.2" x509-parser = "0.15.0" sha256 = "1.0.3" tonic = { version = "0.6.2", features=['compression', 'tls'] } tokio = { version = "1.17.0", features = ["macros", "rt-multi-thread", "fs", "process"] } warp = "0.3" npyz = "0.7.3" base64 = "0.21.0" histogram = "0.6.9" tch = {version = "0.10.3", optional = true} tensorflow = { version = "0.18.0", optional = true } once_cell = {version = "1.17.1"} ndarray = "0.15" serde = "1.0.154" serde_json = "1.0.94" dr_transform = { path = "../dr_transform"} [build-dependencies] tonic-build = {version = "0.6.2", features=['prost', "compression"] } [profile.release] debug = true [dev-dependencies] ndarray-rand = "0.14.0" tokio-test = "*" assert_cmd = "2.0" criterion = "0.4.0" ================================================ FILE: navi/navi/build.rs ================================================ fn main() -> Result<(), Box> { //::compile_protos("proto/tensorflow_serving/apis/prediction_service.proto")?; tonic_build::configure().compile( &[ "proto/tensorflow_serving/apis/prediction_service.proto", "proto/tensorflow/core/protobuf/config.proto", "proto/tensorflow_serving/apis/prediction_log.proto", "proto/kfserving/grpc_predict_v2.proto", ], &["proto"], )?; Ok(()) } ================================================ FILE: navi/navi/proto/kfserving/grpc_predict_v2.proto ================================================ // Copyright 2020 kubeflow.org. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package inference; // Inference Server GRPC endpoints. service GRPCInferenceService { // The ServerLive API indicates if the inference server is able to receive // and respond to metadata and inference requests. rpc ServerLive(ServerLiveRequest) returns (ServerLiveResponse) {} // The ServerReady API indicates if the server is ready for inferencing. rpc ServerReady(ServerReadyRequest) returns (ServerReadyResponse) {} // The ModelReady API indicates if a specific model is ready for inferencing. rpc ModelReady(ModelReadyRequest) returns (ModelReadyResponse) {} // The ServerMetadata API provides information about the server. Errors are // indicated by the google.rpc.Status returned for the request. The OK code // indicates success and other codes indicate failure. rpc ServerMetadata(ServerMetadataRequest) returns (ServerMetadataResponse) {} // The per-model metadata API provides information about a model. Errors are // indicated by the google.rpc.Status returned for the request. The OK code // indicates success and other codes indicate failure. rpc ModelMetadata(ModelMetadataRequest) returns (ModelMetadataResponse) {} // The ModelInfer API performs inference using the specified model. Errors are // indicated by the google.rpc.Status returned for the request. The OK code // indicates success and other codes indicate failure. rpc ModelInfer(ModelInferRequest) returns (ModelInferResponse) {} } message ServerLiveRequest {} message ServerLiveResponse { // True if the inference server is live, false if not live. bool live = 1; } message ServerReadyRequest {} message ServerReadyResponse { // True if the inference server is ready, false if not ready. bool ready = 1; } message ModelReadyRequest { // The name of the model to check for readiness. string name = 1; // The version of the model to check for readiness. If not given the // server will choose a version based on the model and internal policy. string version = 2; } message ModelReadyResponse { // True if the model is ready, false if not ready. bool ready = 1; } message ServerMetadataRequest {} message ServerMetadataResponse { // The server name. string name = 1; // The server version. string version = 2; // The extensions supported by the server. repeated string extensions = 3; } message ModelMetadataRequest { // The name of the model. string name = 1; // The version of the model to check for readiness. If not given the // server will choose a version based on the model and internal policy. string version = 2; } message ModelMetadataResponse { // Metadata for a tensor. message TensorMetadata { // The tensor name. string name = 1; // The tensor data type. string datatype = 2; // The tensor shape. A variable-size dimension is represented // by a -1 value. repeated int64 shape = 3; } // The model name. string name = 1; // The versions of the model available on the server. repeated string versions = 2; // The model's platform. See Platforms. string platform = 3; // The model's inputs. repeated TensorMetadata inputs = 4; // The model's outputs. repeated TensorMetadata outputs = 5; } message ModelInferRequest { // An input tensor for an inference request. message InferInputTensor { // The tensor name. string name = 1; // The tensor data type. string datatype = 2; // The tensor shape. repeated int64 shape = 3; // Optional inference input tensor parameters. map parameters = 4; // The tensor contents using a data-type format. This field must // not be specified if "raw" tensor contents are being used for // the inference request. InferTensorContents contents = 5; } // An output tensor requested for an inference request. message InferRequestedOutputTensor { // The tensor name. string name = 1; // Optional requested output tensor parameters. map parameters = 2; } // The name of the model to use for inferencing. string model_name = 1; // The version of the model to use for inference. If not given the // server will choose a version based on the model and internal policy. string model_version = 2; // Optional identifier for the request. If specified will be // returned in the response. string id = 3; // Optional inference parameters. map parameters = 4; // The input tensors for the inference. repeated InferInputTensor inputs = 5; // The requested output tensors for the inference. Optional, if not // specified all outputs produced by the model will be returned. repeated InferRequestedOutputTensor outputs = 6; // The data contained in an input tensor can be represented in "raw" // bytes form or in the repeated type that matches the tensor's data // type. To use the raw representation 'raw_input_contents' must be // initialized with data for each tensor in the same order as // 'inputs'. For each tensor, the size of this content must match // what is expected by the tensor's shape and data type. The raw // data must be the flattened, one-dimensional, row-major order of // the tensor elements without any stride or padding between the // elements. Note that the FP16 and BF16 data types must be represented as // raw content as there is no specific data type for a 16-bit float type. // // If this field is specified then InferInputTensor::contents must // not be specified for any input tensor. repeated bytes raw_input_contents = 7; } message ModelInferResponse { // An output tensor returned for an inference request. message InferOutputTensor { // The tensor name. string name = 1; // The tensor data type. string datatype = 2; // The tensor shape. repeated int64 shape = 3; // Optional output tensor parameters. map parameters = 4; // The tensor contents using a data-type format. This field must // not be specified if "raw" tensor contents are being used for // the inference response. InferTensorContents contents = 5; } // The name of the model used for inference. string model_name = 1; // The version of the model used for inference. string model_version = 2; // The id of the inference request if one was specified. string id = 3; // Optional inference response parameters. map parameters = 4; // The output tensors holding inference results. repeated InferOutputTensor outputs = 5; // The data contained in an output tensor can be represented in // "raw" bytes form or in the repeated type that matches the // tensor's data type. To use the raw representation 'raw_output_contents' // must be initialized with data for each tensor in the same order as // 'outputs'. For each tensor, the size of this content must match // what is expected by the tensor's shape and data type. The raw // data must be the flattened, one-dimensional, row-major order of // the tensor elements without any stride or padding between the // elements. Note that the FP16 and BF16 data types must be represented as // raw content as there is no specific data type for a 16-bit float type. // // If this field is specified then InferOutputTensor::contents must // not be specified for any output tensor. repeated bytes raw_output_contents = 6; } // An inference parameter value. The Parameters message describes a // “name”/”value” pair, where the “name” is the name of the parameter // and the “value” is a boolean, integer, or string corresponding to // the parameter. message InferParameter { // The parameter value can be a string, an int64, a boolean // or a message specific to a predefined parameter. oneof parameter_choice { // A boolean parameter value. bool bool_param = 1; // An int64 parameter value. int64 int64_param = 2; // A string parameter value. string string_param = 3; } } // The data contained in a tensor represented by the repeated type // that matches the tensor's data type. Protobuf oneof is not used // because oneofs cannot contain repeated fields. message InferTensorContents { // Representation for BOOL data type. The size must match what is // expected by the tensor's shape. The contents must be the flattened, // one-dimensional, row-major order of the tensor elements. repeated bool bool_contents = 1; // Representation for INT8, INT16, and INT32 data types. The size // must match what is expected by the tensor's shape. The contents // must be the flattened, one-dimensional, row-major order of the // tensor elements. repeated int32 int_contents = 2; // Representation for INT64 data types. The size must match what // is expected by the tensor's shape. The contents must be the // flattened, one-dimensional, row-major order of the tensor elements. repeated int64 int64_contents = 3; // Representation for UINT8, UINT16, and UINT32 data types. The size // must match what is expected by the tensor's shape. The contents // must be the flattened, one-dimensional, row-major order of the // tensor elements. repeated uint32 uint_contents = 4; // Representation for UINT64 data types. The size must match what // is expected by the tensor's shape. The contents must be the // flattened, one-dimensional, row-major order of the tensor elements. repeated uint64 uint64_contents = 5; // Representation for FP32 data type. The size must match what is // expected by the tensor's shape. The contents must be the flattened, // one-dimensional, row-major order of the tensor elements. repeated float fp32_contents = 6; // Representation for FP64 data type. The size must match what is // expected by the tensor's shape. The contents must be the flattened, // one-dimensional, row-major order of the tensor elements. repeated double fp64_contents = 7; // Representation for BYTES data type. The size must match what is // expected by the tensor's shape. The contents must be the flattened, // one-dimensional, row-major order of the tensor elements. repeated bytes bytes_contents = 8; } ================================================ FILE: navi/navi/proto/tensorflow/core/example/example.proto ================================================ // Protocol messages for describing input data Examples for machine learning // model training or inference. syntax = "proto3"; package tensorflow; import "tensorflow/core/example/feature.proto"; option cc_enable_arenas = true; option java_outer_classname = "ExampleProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.example"; option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/example"; // LINT.IfChange // An Example is a mostly-normalized data format for storing data for // training and inference. It contains a key-value store (features); where // each key (string) maps to a Feature message (which is oneof packed BytesList, // FloatList, or Int64List). This flexible and compact format allows the // storage of large amounts of typed data, but requires that the data shape // and use be determined by the configuration files and parsers that are used to // read and write this format. That is, the Example is mostly *not* a // self-describing format. In TensorFlow, Examples are read in row-major // format, so any configuration that describes data with rank-2 or above // should keep this in mind. For example, to store an M x N matrix of Bytes, // the BytesList must contain M*N bytes, with M rows of N contiguous values // each. That is, the BytesList value must store the matrix as: // .... row 0 .... .... row 1 .... // ........... // ... row M-1 .... // // An Example for a movie recommendation application: // features { // feature { // key: "age" // value { float_list { // value: 29.0 // }} // } // feature { // key: "movie" // value { bytes_list { // value: "The Shawshank Redemption" // value: "Fight Club" // }} // } // feature { // key: "movie_ratings" // value { float_list { // value: 9.0 // value: 9.7 // }} // } // feature { // key: "suggestion" // value { bytes_list { // value: "Inception" // }} // } // # Note that this feature exists to be used as a label in training. // # E.g., if training a logistic regression model to predict purchase // # probability in our learning tool we would set the label feature to // # "suggestion_purchased". // feature { // key: "suggestion_purchased" // value { float_list { // value: 1.0 // }} // } // # Similar to "suggestion_purchased" above this feature exists to be used // # as a label in training. // # E.g., if training a linear regression model to predict purchase // # price in our learning tool we would set the label feature to // # "purchase_price". // feature { // key: "purchase_price" // value { float_list { // value: 9.99 // }} // } // } // // A conformant Example data set obeys the following conventions: // - If a Feature K exists in one example with data type T, it must be of // type T in all other examples when present. It may be omitted. // - The number of instances of Feature K list data may vary across examples, // depending on the requirements of the model. // - If a Feature K doesn't exist in an example, a K-specific default will be // used, if configured. // - If a Feature K exists in an example but contains no items, the intent // is considered to be an empty tensor and no default will be used. message Example { Features features = 1; } // A SequenceExample is an Example representing one or more sequences, and // some context. The context contains features which apply to the entire // example. The feature_lists contain a key, value map where each key is // associated with a repeated set of Features (a FeatureList). // A FeatureList thus represents the values of a feature identified by its key // over time / frames. // // Below is a SequenceExample for a movie recommendation application recording a // sequence of ratings by a user. The time-independent features ("locale", // "age", "favorites") describing the user are part of the context. The sequence // of movies the user rated are part of the feature_lists. For each movie in the // sequence we have information on its name and actors and the user's rating. // This information is recorded in three separate feature_list(s). // In the example below there are only two movies. All three feature_list(s), // namely "movie_ratings", "movie_names", and "actors" have a feature value for // both movies. Note, that "actors" is itself a bytes_list with multiple // strings per movie. // // context: { // feature: { // key : "locale" // value: { // bytes_list: { // value: [ "pt_BR" ] // } // } // } // feature: { // key : "age" // value: { // float_list: { // value: [ 19.0 ] // } // } // } // feature: { // key : "favorites" // value: { // bytes_list: { // value: [ "Majesty Rose", "Savannah Outen", "One Direction" ] // } // } // } // } // feature_lists: { // feature_list: { // key : "movie_ratings" // value: { // feature: { // float_list: { // value: [ 4.5 ] // } // } // feature: { // float_list: { // value: [ 5.0 ] // } // } // } // } // feature_list: { // key : "movie_names" // value: { // feature: { // bytes_list: { // value: [ "The Shawshank Redemption" ] // } // } // feature: { // bytes_list: { // value: [ "Fight Club" ] // } // } // } // } // feature_list: { // key : "actors" // value: { // feature: { // bytes_list: { // value: [ "Tim Robbins", "Morgan Freeman" ] // } // } // feature: { // bytes_list: { // value: [ "Brad Pitt", "Edward Norton", "Helena Bonham Carter" ] // } // } // } // } // } // // A conformant SequenceExample data set obeys the following conventions: // // Context: // - All conformant context features K must obey the same conventions as // a conformant Example's features (see above). // Feature lists: // - A FeatureList L may be missing in an example; it is up to the // parser configuration to determine if this is allowed or considered // an empty list (zero length). // - If a FeatureList L exists, it may be empty (zero length). // - If a FeatureList L is non-empty, all features within the FeatureList // must have the same data type T. Even across SequenceExamples, the type T // of the FeatureList identified by the same key must be the same. An entry // without any values may serve as an empty feature. // - If a FeatureList L is non-empty, it is up to the parser configuration // to determine if all features within the FeatureList must // have the same size. The same holds for this FeatureList across multiple // examples. // - For sequence modeling, e.g.: // http://colah.github.io/posts/2015-08-Understanding-LSTMs/ // https://github.com/tensorflow/nmt // the feature lists represent a sequence of frames. // In this scenario, all FeatureLists in a SequenceExample have the same // number of Feature messages, so that the ith element in each FeatureList // is part of the ith frame (or time step). // Examples of conformant and non-conformant examples' FeatureLists: // // Conformant FeatureLists: // feature_lists: { feature_list: { // key: "movie_ratings" // value: { feature: { float_list: { value: [ 4.5 ] } } // feature: { float_list: { value: [ 5.0 ] } } } // } } // // Non-conformant FeatureLists (mismatched types): // feature_lists: { feature_list: { // key: "movie_ratings" // value: { feature: { float_list: { value: [ 4.5 ] } } // feature: { int64_list: { value: [ 5 ] } } } // } } // // Conditionally conformant FeatureLists, the parser configuration determines // if the feature sizes must match: // feature_lists: { feature_list: { // key: "movie_ratings" // value: { feature: { float_list: { value: [ 4.5 ] } } // feature: { float_list: { value: [ 5.0, 6.0 ] } } } // } } // // Conformant pair of SequenceExample // feature_lists: { feature_list: { // key: "movie_ratings" // value: { feature: { float_list: { value: [ 4.5 ] } } // feature: { float_list: { value: [ 5.0 ] } } } // } } // and: // feature_lists: { feature_list: { // key: "movie_ratings" // value: { feature: { float_list: { value: [ 4.5 ] } } // feature: { float_list: { value: [ 5.0 ] } } // feature: { float_list: { value: [ 2.0 ] } } } // } } // // Conformant pair of SequenceExample // feature_lists: { feature_list: { // key: "movie_ratings" // value: { feature: { float_list: { value: [ 4.5 ] } } // feature: { float_list: { value: [ 5.0 ] } } } // } } // and: // feature_lists: { feature_list: { // key: "movie_ratings" // value: { } // } } // // Conditionally conformant pair of SequenceExample, the parser configuration // determines if the second feature_lists is consistent (zero-length) or // invalid (missing "movie_ratings"): // feature_lists: { feature_list: { // key: "movie_ratings" // value: { feature: { float_list: { value: [ 4.5 ] } } // feature: { float_list: { value: [ 5.0 ] } } } // } } // and: // feature_lists: { } // // Non-conformant pair of SequenceExample (mismatched types) // feature_lists: { feature_list: { // key: "movie_ratings" // value: { feature: { float_list: { value: [ 4.5 ] } } // feature: { float_list: { value: [ 5.0 ] } } } // } } // and: // feature_lists: { feature_list: { // key: "movie_ratings" // value: { feature: { int64_list: { value: [ 4 ] } } // feature: { int64_list: { value: [ 5 ] } } // feature: { int64_list: { value: [ 2 ] } } } // } } // // Conditionally conformant pair of SequenceExample; the parser configuration // determines if the feature sizes must match: // feature_lists: { feature_list: { // key: "movie_ratings" // value: { feature: { float_list: { value: [ 4.5 ] } } // feature: { float_list: { value: [ 5.0 ] } } } // } } // and: // feature_lists: { feature_list: { // key: "movie_ratings" // value: { feature: { float_list: { value: [ 4.0 ] } } // feature: { float_list: { value: [ 5.0, 3.0 ] } } // } } message SequenceExample { Features context = 1; FeatureLists feature_lists = 2; } // LINT.ThenChange( // https://www.tensorflow.org/code/tensorflow/python/training/training.py) ================================================ FILE: navi/navi/proto/tensorflow/core/example/feature.proto ================================================ // Protocol messages for describing features for machine learning model // training or inference. // // There are three base Feature types: // - bytes // - float // - int64 // // A Feature contains Lists which may hold zero or more values. These // lists are the base values BytesList, FloatList, Int64List. // // Features are organized into categories by name. The Features message // contains the mapping from name to Feature. // // Example Features for a movie recommendation application: // feature { // key: "age" // value { float_list { // value: 29.0 // }} // } // feature { // key: "movie" // value { bytes_list { // value: "The Shawshank Redemption" // value: "Fight Club" // }} // } // feature { // key: "movie_ratings" // value { float_list { // value: 9.0 // value: 9.7 // }} // } // feature { // key: "suggestion" // value { bytes_list { // value: "Inception" // }} // } // feature { // key: "suggestion_purchased" // value { int64_list { // value: 1 // }} // } // feature { // key: "purchase_price" // value { float_list { // value: 9.99 // }} // } // syntax = "proto3"; package tensorflow; option cc_enable_arenas = true; option java_outer_classname = "FeatureProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.example"; option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/example"; // LINT.IfChange // Containers to hold repeated fundamental values. message BytesList { repeated bytes value = 1; } message FloatList { repeated float value = 1 [packed = true]; } message Int64List { repeated int64 value = 1 [packed = true]; } // Containers for non-sequential data. message Feature { // Each feature can be exactly one kind. oneof kind { BytesList bytes_list = 1; FloatList float_list = 2; Int64List int64_list = 3; } } message Features { // Map from feature name to feature. map feature = 1; } // Containers for sequential data. // // A FeatureList contains lists of Features. These may hold zero or more // Feature values. // // FeatureLists are organized into categories by name. The FeatureLists message // contains the mapping from name to FeatureList. // message FeatureList { repeated Feature feature = 1; } message FeatureLists { // Map from feature name to feature list. map feature_list = 1; } // LINT.ThenChange( // https://www.tensorflow.org/code/tensorflow/python/training/training.py) ================================================ FILE: navi/navi/proto/tensorflow/core/framework/allocation_description.proto ================================================ syntax = "proto3"; package tensorflow; option cc_enable_arenas = true; option java_outer_classname = "AllocationDescriptionProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/allocation_description_go_proto"; message AllocationDescription { // Total number of bytes requested int64 requested_bytes = 1; // Total number of bytes allocated if known int64 allocated_bytes = 2; // Name of the allocator used string allocator_name = 3; // Identifier of the allocated buffer if known int64 allocation_id = 4; // Set if this tensor only has one remaining reference bool has_single_reference = 5; // Address of the allocation. uint64 ptr = 6; } ================================================ FILE: navi/navi/proto/tensorflow/core/framework/api_def.proto ================================================ // Defines the text format for including per-op API definition and // overrides for client language op code generators. syntax = "proto3"; package tensorflow; import "tensorflow/core/framework/attr_value.proto"; option cc_enable_arenas = true; option java_outer_classname = "ApiDefProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/api_def_go_proto"; // Used to specify and override the default API & behavior in the // generated code for client languages, from what you would get from // the OpDef alone. There will be a set of ApiDefs that are common // to all client languages, and another set per client language. // The per-client-language ApiDefs will inherit values from the // common ApiDefs which it can either replace or modify. // // We separate the API definition from the OpDef so we can evolve the // API while remaining backwards compatible when interpreting old // graphs. Overrides go in an "api_def.pbtxt" file with a text-format // ApiDefs message. // // WARNING: Be *very* careful changing the API for any existing op -- // you can change the semantics of existing code. These changes may // need to wait until a major release of TensorFlow to avoid breaking // our compatibility promises. message ApiDef { // Name of the op (in the OpDef) to specify the API for. string graph_op_name = 1; // If this op is deprecated, set deprecation message to the message // that should be logged when this op is used. // The message should indicate alternative op to use, if any. string deprecation_message = 12; // Major version when the op will be deleted. For e.g. set this // value to 2 if op API should be removed in TensorFlow 2.0 and // deprecated in versions before that. int32 deprecation_version = 13; enum Visibility { // Normally this is "VISIBLE" unless you are inheriting a // different value from another ApiDef. DEFAULT_VISIBILITY = 0; // Publicly visible in the API. VISIBLE = 1; // Do not include this op in the generated API. If visibility is // set to 'SKIP', other fields are ignored for this op. SKIP = 2; // Hide this op by putting it into an internal namespace (or whatever // is appropriate in the target language). HIDDEN = 3; } Visibility visibility = 2; // If you specify any endpoint, this will replace all of the // inherited endpoints. The first endpoint should be the // "canonical" endpoint, and should not be deprecated (unless all // endpoints are deprecated). message Endpoint { // Name should be either like "CamelCaseName" or // "Package.CamelCaseName". Client-language-specific ApiDefs may // use a snake_case convention instead of CamelCase. string name = 1; // Set if this endpoint is deprecated. If set to true, a message suggesting // to use a non-deprecated endpoint instead will be printed. If all // endpoints are deprecated, set deprecation_message in ApiDef instead. bool deprecated = 3; // Major version when an endpoint will be deleted. For e.g. set this // value to 2 if endpoint should be removed in TensorFlow 2.0 and // deprecated in versions before that. int32 deprecation_version = 4; } repeated Endpoint endpoint = 3; message Arg { string name = 1; // Change the name used to access this arg in the API from what // is used in the GraphDef. Note that these names in `backticks` // will also be replaced in the summary & description fields. string rename_to = 2; // Note: this will replace any inherited arg doc. There is no // current way of modifying arg descriptions (other than replacing // them entirely) as can be done with op descriptions. string description = 3; } repeated Arg in_arg = 4; repeated Arg out_arg = 5; // List of original in_arg names to specify new argument order. // Length of arg_order should be either empty to keep current order // or match size of in_arg. repeated string arg_order = 11; // Description of the graph-construction-time configuration of this // Op. That is to say, this describes the attr fields that will // be specified in the NodeDef. message Attr { string name = 1; // Change the name used to access this attr in the API from what // is used in the GraphDef. Note that these names in `backticks` // will also be replaced in the summary & description fields. string rename_to = 2; // Specify a new default value to use for this attr. This default // will be used when creating new graphs, as opposed to the // default in the OpDef, which will be used when interpreting old // GraphDefs. AttrValue default_value = 3; // Note: this will replace any inherited attr doc, there is no current // way of modifying attr descriptions as can be done with op descriptions. string description = 4; } repeated Attr attr = 6; // One-line human-readable description of what the Op does. string summary = 7; // Additional, longer human-readable description of what the Op does. string description = 8; // Modify an existing/inherited description by adding text to the beginning // or end. string description_prefix = 9; string description_suffix = 10; } message ApiDefs { repeated ApiDef op = 1; } ================================================ FILE: navi/navi/proto/tensorflow/core/framework/attr_value.proto ================================================ syntax = "proto3"; package tensorflow; import "tensorflow/core/framework/tensor.proto"; import "tensorflow/core/framework/tensor_shape.proto"; import "tensorflow/core/framework/types.proto"; option cc_enable_arenas = true; option java_outer_classname = "AttrValueProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/attr_value_go_proto"; // Protocol buffer representing the value for an attr used to configure an Op. // Comment indicates the corresponding attr type. Only the field matching the // attr type may be filled. message AttrValue { // LINT.IfChange message ListValue { repeated bytes s = 2; // "list(string)" repeated int64 i = 3 [packed = true]; // "list(int)" repeated float f = 4 [packed = true]; // "list(float)" repeated bool b = 5 [packed = true]; // "list(bool)" repeated DataType type = 6 [packed = true]; // "list(type)" repeated TensorShapeProto shape = 7; // "list(shape)" repeated TensorProto tensor = 8; // "list(tensor)" repeated NameAttrList func = 9; // "list(attr)" } // LINT.ThenChange(https://www.tensorflow.org/code/tensorflow/c/c_api.cc) oneof value { bytes s = 2; // "string" int64 i = 3; // "int" float f = 4; // "float" bool b = 5; // "bool" DataType type = 6; // "type" TensorShapeProto shape = 7; // "shape" TensorProto tensor = 8; // "tensor" ListValue list = 1; // any "list(...)" // "func" represents a function. func.name is a function's name or // a primitive op's name. func.attr.first is the name of an attr // defined for that function. func.attr.second is the value for // that attr in the instantiation. NameAttrList func = 10; // This is a placeholder only used in nodes defined inside a // function. It indicates the attr value will be supplied when // the function is instantiated. For example, let us suppose a // node "N" in function "FN". "N" has an attr "A" with value // placeholder = "foo". When FN is instantiated with attr "foo" // set to "bar", the instantiated node N's attr A will have been // given the value "bar". string placeholder = 9; } } // A list of attr names and their values. The whole list is attached // with a string name. E.g., MatMul[T=float]. message NameAttrList { string name = 1; map attr = 2; } ================================================ FILE: navi/navi/proto/tensorflow/core/framework/cost_graph.proto ================================================ syntax = "proto3"; package tensorflow; import "tensorflow/core/framework/tensor_shape.proto"; import "tensorflow/core/framework/types.proto"; option cc_enable_arenas = true; option java_outer_classname = "CostGraphProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/cost_graph_go_proto"; message CostGraphDef { message Node { // The name of the node. Names are globally unique. string name = 1; // The device of the node. Can be empty if the node is mapped to the // default partition or partitioning hasn't been run yet. string device = 2; // The id of the node. Node ids are only unique inside a partition. int32 id = 3; // Inputs of this node. They must be executed before this node can be // executed. An input is a particular output of another node, specified // by the node id and the output index. message InputInfo { int32 preceding_node = 1; int32 preceding_port = 2; } repeated InputInfo input_info = 4; // Outputs of this node. message OutputInfo { int64 size = 1; // If >= 0, the output is an alias of an input. Note that an alias input // may itself be an alias. The algorithm will therefore need to follow // those pointers. int64 alias_input_port = 2; TensorShapeProto shape = 3; DataType dtype = 4; } repeated OutputInfo output_info = 5; // Temporary memory used by this node. int64 temporary_memory_size = 6; // Persistent memory used by this node. int64 persistent_memory_size = 12; int64 host_temp_memory_size = 10 [deprecated = true]; int64 device_temp_memory_size = 11 [deprecated = true]; int64 device_persistent_memory_size = 16 [deprecated = true]; // Estimate of the computational cost of this node, in microseconds. int64 compute_cost = 9; // Analytical estimate of the computational cost of this node, in // microseconds. int64 compute_time = 14; // Analytical estimate of the memory access cost of this node, in // microseconds. int64 memory_time = 15; // If true, the output is permanent: it can't be discarded, because this // node is part of the "final output". Nodes may depend on final nodes. bool is_final = 7; // Ids of the control inputs for this node. repeated int32 control_input = 8; // Are the costs inaccurate? bool inaccurate = 17; } repeated Node node = 1; // Total cost of this graph, typically used for balancing decisions. message AggregatedCost { // Aggregated cost value. float cost = 1; // Aggregated cost dimension (e.g. 'memory', 'compute', 'network'). string dimension = 2; } repeated AggregatedCost cost = 2; } ================================================ FILE: navi/navi/proto/tensorflow/core/framework/dataset_metadata.proto ================================================ syntax = "proto3"; package tensorflow.data; option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/dataset_metadata_go_proto"; // next: 2 message Metadata { bytes name = 1; } ================================================ FILE: navi/navi/proto/tensorflow/core/framework/dataset_options.proto ================================================ syntax = "proto3"; package tensorflow.data; import "tensorflow/core/framework/model.proto"; option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/dataset_options_go_proto"; // Represents the type of auto-sharding we enable. enum AutoShardPolicy { // AUTO: Attempts FILE-based sharding, falling back to DATA-based sharding. AUTO = 0; // FILE: Shards by input files (i.e. each worker will get a set of files to // process). When this option is selected, make sure that there is at least as // many files as workers. If there are fewer input files than workers, a // runtime error will be raised. FILE = 1; // DATA: Shards by elements produced by the dataset. Each worker will process // the whole dataset and discard the portion that is not for itself. Note that // for this mode to correctly partitions the dataset elements, the dataset // needs to produce elements in a deterministic order. DATA = 2; // HINT: Looks for the presence of `shard(SHARD_HINT, ...)` which is treated // as a placeholder to replace with `shard(num_workers, worker_index)`. HINT = 3; // OFF: No sharding will be performed. OFF = -1; } // next: 5 message AutotuneOptions { // Whether to automatically tune performance knobs. oneof optional_enabled { bool enabled = 1; } // When autotuning is enabled (through autotune), determines the CPU budget to // use. Values greater than the number of schedulable CPU cores are allowed // but may result in CPU contention. oneof optional_cpu_budget { int32 cpu_budget = 2; } // When autotuning is enabled (through autotune), determines the RAM budget to // use. Values greater than the available RAM in bytes may result in OOM. If // 0, defaults to half of the available RAM in bytes. oneof optional_ram_budget { int64 ram_budget = 3; } // When autotuning is enabled (through autotune), determines the algorithm to // use. If not explicitly set by user, autotuning will follow HILL_CLIMB // algorithm but has more flexibility to tune parameters more aggressively, // in which case the behavior is implementation specific and may change over // time. oneof optional_autotune_algorithm { model.AutotuneAlgorithm autotune_algorithm = 4; } } // next: 2 message CardinalityOptions { enum ComputeLevel { CARDINALITY_COMPUTE_UNSPECIFIED = 0; // Cardinality will only be computed if it can be determined in a cheap // manner (ie. without reading from file sources). If the cardinality would // be nontrivial to compute, Cardinality() will return UNKNOWN_CARDINALITY. CARDINALITY_COMPUTE_LOW = 1; // Moderate effort will be made to determine cardinality, such as reading // index data from source files. If significant work is needed to compute // cardinality (e.g. reading entire source file contents or executing user // defined functions), Cardinality() will return UNKNOWN_CARDINALITY. CARDINALITY_COMPUTE_MODERATE = 2; } ComputeLevel compute_level = 1; } // next: 3 message DistributeOptions { AutoShardPolicy auto_shard_policy = 1; // The number of devices attached to this input pipeline. oneof optional_num_devices { int32 num_devices = 2; } } // next: 18 message OptimizationOptions { // Whether to apply default graph optimizations. If False, only graph // optimizations that have been explicitly enabled will be applied. oneof optional_apply_default_optimizations { bool apply_default_optimizations = 1; } reserved 2; reserved 3; reserved 4; reserved 5; // Whether to fuse filter transformations. oneof optional_filter_fusion { bool filter_fusion = 6; } // NOTE: field id 7 deleted in June 2021. reserved 7; // NOTE: field id 8 deleted in June 2021. reserved 8; // Whether to fuse map and batch transformations. oneof optional_map_and_batch_fusion { bool map_and_batch_fusion = 9; } // Whether to fuse map and filter transformations. oneof optional_map_and_filter_fusion { bool map_and_filter_fusion = 10; } // Whether to fuse map transformations. oneof optional_map_fusion { bool map_fusion = 11; } // Whether to parallelize stateless map transformations. oneof optional_map_parallelization { bool map_parallelization = 12; } // NOTE: field id 13 deleted in June 2021. reserved 13; // Whether to eliminate no-op transformations. oneof optional_noop_elimination { bool noop_elimination = 14; } // Whether to parallelize copying of batch elements. This optimization is // highly experimental and can cause performance degradation (e.g. when the // parallelization overhead exceeds the benefits of performing the data copies // in parallel). You should only enable this optimization if a) your input // pipeline is bottlenecked on batching and b) you have validated that this // optimization improves performance. oneof optional_parallel_batch { bool parallel_batch = 15; } // Field id 16 was removed in 06/2021. reserved 16; // Whether to fuse shuffle and repeat transformations. oneof optional_shuffle_and_repeat_fusion { bool shuffle_and_repeat_fusion = 17; } } // next: 3 message ThreadingOptions { // If set, it overrides the maximum degree of intra-op parallelism. oneof optional_max_intra_op_parallelism { int32 max_intra_op_parallelism = 1; } // If set, the dataset will use a private threadpool of the given size. oneof optional_private_threadpool_size { int32 private_threadpool_size = 2; } } // Represents how to handle external state during serialization. enum ExternalStatePolicy { POLICY_WARN = 0; POLICY_IGNORE = 1; POLICY_FAIL = 2; } // Message stored with Dataset objects to control how datasets are processed and // optimized. // // next: 8 message Options { // Whether the outputs need to be produced in deterministic order. oneof optional_deterministic { bool deterministic = 1; } // The distribution strategy options associated with the dataset. AutotuneOptions autotune_options = 7; // The distribution strategy options associated with the dataset. DistributeOptions distribute_options = 2; // The optimization options associated with the dataset. OptimizationOptions optimization_options = 3; // Whether to introduce 'slack' in the last `prefetch` of the input pipeline, // if it exists. This may reduce CPU contention with accelerator host-side // activity at the start of a step. The slack frequency is determined by the // number of devices attached to this input pipeline. oneof optional_slack { bool slack = 4; } // The threading options associated with the dataset. ThreadingOptions threading_options = 5; // This option can be used to override the default policy for how to handle // external state when serializing a dataset or checkpointing its iterator. // There are three settings available - IGNORE: External state is ignored // without a warning; WARN: External state is ignored and a warning is logged; // FAIL: External state results in an error. oneof optional_external_state_policy { ExternalStatePolicy external_state_policy = 6; } } ================================================ FILE: navi/navi/proto/tensorflow/core/framework/device_attributes.proto ================================================ syntax = "proto3"; package tensorflow; option cc_enable_arenas = true; option java_outer_classname = "DeviceAttributesProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/device_attributes_go_proto"; message InterconnectLink { int32 device_id = 1; string type = 2; int32 strength = 3; } message LocalLinks { repeated InterconnectLink link = 1; } message DeviceLocality { // Optional bus locality of device. Default value of 0 means // no specific locality. Specific localities are indexed from 1. int32 bus_id = 1; // Optional NUMA locality of device. int32 numa_node = 2; // Optional local interconnect links to other devices. LocalLinks links = 3; } message DeviceAttributes { // Fully specified name of the device within a cluster. string name = 1; // String representation of device_type. string device_type = 2; // Memory capacity of device in bytes. int64 memory_limit = 4; // Platform-specific data about device that may be useful // for supporting efficient data transfers. DeviceLocality locality = 5; // A device is assigned a global unique number each time it is // initialized. "incarnation" should never be 0. fixed64 incarnation = 6; // String representation of the physical device that this device maps to. string physical_device_desc = 7; // A physical device ID for use in XLA DeviceAssignments, unique across // clients in a multi-client setup. Set to -1 if unavailable, non-negative // otherwise. int64 xla_global_id = 8; } ================================================ FILE: navi/navi/proto/tensorflow/core/framework/full_type.proto ================================================ syntax = "proto3"; package tensorflow; option cc_enable_arenas = true; option java_outer_classname = "FullTypeProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/full_type_go_proto"; // Experimental. Represents the complete type information of a TensorFlow value. enum FullTypeId { // The default represents an uninitialized values. TFT_UNSET = 0; // Type symbols. Used to construct more complex type expressions like // algebraic data types. // Type variables may serve as placeholder for any other type ID in type // templates. // // Examples: // TFT_DATASET[TFT_VAR["T"]] is a Dataset returning a type indicated by "T". // TFT_TENSOR[TFT_VAR["T"]] is a Tensor of n element type indicated by "T". // TFT_TENSOR[TFT_VAR["T"]], TFT_TENSOR[TFT_VAR["T"]] are two tensors of // identical element types. // TFT_TENSOR[TFT_VAR["P"]], TFT_TENSOR[TFT_VAR["Q"]] are two tensors of // independent element types. // TFT_VAR = 1; // Wildcard type. Describes a parameter of unknown type. In TensorFlow, that // can mean either a "Top" type (accepts any type), or a dynamically typed // object whose type is unknown in context. // Important: "unknown" does not necessarily mean undeterminable! TFT_ANY = 2; // The algebraic product type. This is an algebraic type that may be used just // for logical grouping. Not to confused with TFT_TUPLE which describes a // concrete object of several elements. // // Example: // TFT_DATASET[TFT_PRODUCT[TFT_TENSOR[TFT_INT32], TFT_TENSOR[TFT_FLOAT64]]] // is a Dataset producing two tensors, an integer one and a float one. // TFT_PRODUCT = 3; // Represents a named field, with the name stored in the attribute. // // Parametrization: // TFT_NAMED[]{} // * is the type of the field // * is the field name, as string (thpugh can theoretically be an int // as well) // // Example: // TFT_RECORD[ // TFT_NAMED[TFT_TENSOR[TFT_INT32]]{'foo'}, // TFT_NAMED[TFT_TENSOR[TFT_FLOAT32]]{'bar'}, // ] // is a structure with two fields, an int tensor "foo" and a float tensor // "bar". TFT_NAMED = 4; // Template definition. Expands the variables by repeating a template as // arguments of container. // // Parametrization: // TFT_FOR_EACH[,